diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..678dbaa --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,144 @@ +name: CI Build + +on: + push: + branches: + - '**' + pull_request: + branches: + - master + +jobs: + build: + runs-on: windows-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Calculate version + id: version + shell: pwsh + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + $year = (Get-Date).Year + $month = (Get-Date).Month + $startOfMonth = (Get-Date -Day 1 -Hour 0 -Minute 0 -Second 0).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") + + $headers = @{ + Authorization = "Bearer $env:GITHUB_TOKEN" + Accept = "application/vnd.github.v3+json" + } + $url = "https://api.github.com/repos/$env:GITHUB_REPOSITORY/actions/workflows/build.yml/runs?created=>=$startOfMonth&per_page=1" + $response = Invoke-RestMethod -Uri $url -Headers $headers -ErrorAction Stop + $rev = $response.total_count + + $version = "1.$year.$month.$rev" + "VERSION=$version" | Out-File -FilePath $env:GITHUB_OUTPUT -Append + Write-Host "Computed version: $version" + + - name: Patch AssemblyInfo.cs + shell: pwsh + run: | + $version = "${{ steps.version.outputs.VERSION }}" + $file = "XTB\Properties\AssemblyInfo.cs" + (Get-Content $file) ` + -replace 'AssemblyVersion\("[^"]*"\)', "AssemblyVersion(`"$version`")" ` + -replace 'AssemblyFileVersion\("[^"]*"\)', "AssemblyFileVersion(`"$version`")" | + Set-Content $file + Write-Host "Patched $file to version $version" + + - name: Patch nuspec versions + shell: pwsh + run: | + $version = "${{ steps.version.outputs.VERSION }}" + $nuspecs = @( + "XTB\ShuffleRunner.nuspec", + "XTB\ShuffleBuilder.nuspec", + "XTB\ShuffleDeployer.nuspec" + ) + foreach ($nuspec in $nuspecs) { + $path = Resolve-Path $nuspec + [xml]$xml = Get-Content $path + $xml.package.metadata.version = $version + $xml.Save($path) + Write-Host "Patched $nuspec to version $version" + } + + - name: Setup NuGet + uses: NuGet/setup-nuget@v2 + + - name: NuGet restore + run: nuget restore Rappen.XTB.Shuffle.sln + + - name: Setup MSBuild + uses: microsoft/setup-msbuild@v2 + + - name: Build solution (Release) + run: msbuild Rappen.XTB.Shuffle.sln /p:Configuration=Release /p:Platform="Any CPU" /m + + - name: Locate VSTest + id: vstest + shell: pwsh + run: | + $vswhere = Join-Path ${env:ProgramFiles(x86)} "Microsoft Visual Studio\Installer\vswhere.exe" + if (-not (Test-Path $vswhere)) { throw "vswhere.exe not found at $vswhere" } + # No -requires / -latest: a bare -find lists only the instances that actually + # contain the file, so Build Tools counts and the newest instance without the + # test platform cannot win and yield nothing. + $console = & $vswhere -products * ` + -find "Common7\IDE\Extensions\TestPlatform\vstest.console.exe" | Select-Object -First 1 + if (-not $console) { throw "vstest.console.exe not found by vswhere" } + "PATH=$console" | Out-File -FilePath $env:GITHUB_OUTPUT -Append + Write-Host "Found $console" + + - name: Run tests (Release) + shell: pwsh + run: | + & "${{ steps.vstest.outputs.PATH }}" ` + "tests\Xrm.Shuffle.Core.Tests\bin\Release\Xrm.Shuffle.Core.Tests.dll" ` + /Framework:.NETFramework,Version=v4.8 ` + /Logger:"trx;LogFileName=Xrm.Shuffle.Core.Tests.Release.trx" ` + /ResultsDirectory:TestResults + if ($LASTEXITCODE -ne 0) { throw "Tests failed with exit code $LASTEXITCODE" } + + # The product compiles differently in the two configurations: ShuffleDataImport + # logs the match query as FetchXml under #if DEBUG, which sends a message the + # test doubles have to answer. Release alone would let that divergence through, + # green here and red in Visual Studio, which is what happened once already. + - name: Build solution (Debug) + run: msbuild Rappen.XTB.Shuffle.sln /p:Configuration=Debug /p:Platform="Any CPU" /m + + - name: Run tests (Debug) + shell: pwsh + run: | + & "${{ steps.vstest.outputs.PATH }}" ` + "tests\Xrm.Shuffle.Core.Tests\bin\Debug\Xrm.Shuffle.Core.Tests.dll" ` + /Framework:.NETFramework,Version=v4.8 ` + /Logger:"trx;LogFileName=Xrm.Shuffle.Core.Tests.Debug.trx" ` + /ResultsDirectory:TestResults + if ($LASTEXITCODE -ne 0) { throw "Tests failed with exit code $LASTEXITCODE" } + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results-${{ steps.version.outputs.VERSION }} + path: TestResults/*.trx + + - name: NuGet pack + shell: pwsh + run: | + New-Item -ItemType Directory -Force -Path nupkg | Out-Null + nuget pack "XTB\ShuffleRunner.nuspec" -OutputDirectory nupkg + nuget pack "XTB\ShuffleBuilder.nuspec" -OutputDirectory nupkg + nuget pack "XTB\ShuffleDeployer.nuspec" -OutputDirectory nupkg + + - name: Upload nupkg artifacts + uses: actions/upload-artifact@v4 + with: + name: nupkg-${{ steps.version.outputs.VERSION }} + path: nupkg/*.nupkg diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..c041f3d --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,44 @@ +name: Release + +on: + workflow_dispatch: + inputs: + run_id: + description: 'CI run ID whose nupkg artifact to publish (find it in the Actions tab URL)' + required: true + +jobs: + release: + runs-on: windows-latest + + steps: + - name: Download nupkg artifact from CI run + uses: actions/download-artifact@v4 + with: + run-id: ${{ inputs.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + pattern: 'nupkg-*' + path: nupkg + merge-multiple: true + + - name: Resolve version from nupkg filename + id: version + shell: pwsh + run: | + $pkg = Get-ChildItem nupkg\*.nupkg | Select-Object -First 1 + if (-not $pkg) { throw "No .nupkg found in artifact" } + # filename example: Rappen.XrmToolBox.Shuffle.Runner.1.2026.4.5.nupkg + $version = $pkg.BaseName -replace '^Rappen\.XrmToolBox\.Shuffle\.\w+\.', '' + "VERSION=$version" | Out-File -FilePath $env:GITHUB_OUTPUT -Append + Write-Host "Releasing version: $version ($($pkg.Name))" + + - name: Publish to NuGet.org + run: nuget push nupkg\*.nupkg -ApiKey ${{ secrets.NUGET_API_KEY }} -Source https://api.nuget.org/v3/index.json -NonInteractive + + - name: Create GitHub release + uses: softprops/action-gh-release@v2 + with: + tag_name: v${{ steps.version.outputs.VERSION }} + name: Release ${{ steps.version.outputs.VERSION }} + generate_release_notes: true + files: nupkg/*.nupkg diff --git a/.gitignore b/.gitignore index d1a1a0f..3e60359 100644 --- a/.gitignore +++ b/.gitignore @@ -257,3 +257,8 @@ VSIX/ pat.txt test.txt /codealike.json +/.claude/settings.local.json + +# Test run output +TestResults/ +*.trx diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..d8e2159 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,151 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## What This Project Is + +Xrm.Shuffle is an **XrmToolBox plugin suite** for Dataverse/Dynamics CRM. It ships as three separate NuGet packages — Shuffle Builder, Shuffle Runner, Shuffle Deployer — all compiled from a **single assembly** (`Rappen.XTB.Shuffle.dll`). The tools read/write XML-based "ShuffleDefinition" files that describe what Dataverse data or solutions to export or import. + +## Build + +```bash +nuget restore Rappen.XTB.Shuffle.sln +msbuild Rappen.XTB.Shuffle.sln /p:Configuration=Release /p:Platform="Any CPU" /m +``` + +Output goes to `XTB\bin\Release\`. + +Unit tests live in `tests\Xrm.Shuffle.Core.Tests\`. That project imports the same two +`.projitems` as the XTB project, so it compiles the shuffle core without WinForms or +XrmToolBox. Build the **solution** rather than the csproj (the project platform is +`AnyCPU` and the sln does the `Any CPU` mapping), then run the tests with the VSTest +console: + +```bash +vstest.console.exe "tests\Xrm.Shuffle.Core.Tests\bin\Release\Xrm.Shuffle.Core.Tests.dll" /Framework:.NETFramework,Version=v4.8 +``` + +NUnit3TestAdapter arrives through `PackageReference` and is auto-imported, so no +`/TestAdapterPath` is needed. + +Run **both** configurations. `ShuffleDataImport` logs the match query as FetchXml +inside `#if DEBUG`, which sends a message the test doubles have to answer, so Debug +exercises calls Release never makes - and Visual Studio Test Explorer defaults to +Debug. `.github/workflows/build.yml` builds and tests both, and fails the build on a +red test in either. + +Everything outside that project - solution import and export, data export, and anything +that needs a live org - is still validated manually. + +To produce NuGet packages: +```bash +nuget pack "XTB\ShuffleRunner.nuspec" -OutputDirectory nupkg +nuget pack "XTB\ShuffleBuilder.nuspec" -OutputDirectory nupkg +nuget pack "XTB\ShuffleDeployer.nuspec" -OutputDirectory nupkg +``` + +CI runs `.github/workflows/build.yml` / `release.yml` with versioning scheme `1.{year}.{month}.{run_number}`. + +## Architecture + +### Project layout + +``` +XTB/ ← Single .csproj for all three tools + Builder/ ← Visual XML editor for creating definitions + Runner/ ← Executes definitions (export or import) + Deployer/ ← Orchestrates packaged deployments (.cdpkg/.cdzip) +shared/Xrm.Shuffle.Core/ ← Shared Project (.shproj) — compiled directly into XTB, no separate DLL +Xrm.Utils.Core/ ← Git submodule: extensions, fluent API, logging, CSV helpers +``` + +### The Shared Project pattern + +`Xrm.Shuffle.Core` is a **Shared Project** (`.shproj`), not a library. Its files are compiled directly into the main project. All core business logic lives here: + +- **`Shuffler.cs`** — top-level orchestrator; parses the ShuffleDefinition XML, drives block execution, dispatches events +- **`ShuffleDataImport.cs`** — high-performance importer with a capability-detection fallback chain (see below) +- **`ShuffleDataExport.cs`** — query-based exporter; supports filter and FetchXML modes +- **`ShuffleSolutionImport/Export.cs`** — solution package handling +- **`Types.cs`** — core enums (`SerializationType`, `ItemImportResult`, `SolutionImportConditions`) +- **`ShuffleHelper.cs`** — schema validation, DataFileRequired checks, node documentation lookup +- **`Const.cs`** — auto-generated latebound constants for CRM API entities (ImportJob, AsyncOperation) + +The XML schema is `Resources/ShuffleDefinition.xsd`; the corresponding C# class `Resources/ShuffleDefinition.cs` is auto-generated from it. + +### Bulk import strategy — runtime capability detection + +`ShuffleDataImport.cs` detects what the connected environment supports **at runtime per entity** (cached) and walks down this chain: + +1. **UpsertMultiple** (Dataverse online, fastest — bypasses PreRetrieveAll entirely when `UpdateIdentical=true`) +2. **CreateMultiple / UpdateMultiple** (Dataverse online) +3. **ExecuteMultipleRequest** (CRM 9.1 on-premises fallback) +4. **Individual operations** (CRM 8.x fallback) + +Detection uses `sdkmessagefilter` queries. Results are cached per entity name to avoid repeated round-trips. + +### Batching + +Creates, updates and upserts are accumulated into a pending list and flushed in +batches of `BatchSize` (default 1, i.e. no batching). The batch must be flushed early whenever the +next step needs the server to already know about the pending records — before a live +`Match` query, and before `ReplaceGuids` rewrites a record whose lookups point at a +record still in the batch. `IsBatchable` decides what may be batched at all; among +other things it excludes any record carrying `statecode`, `statuscode` or `ownerid`. + +### DeferStateAndOwner + +When `DeferStateAndOwner` is enabled, `statecode`/`statuscode`/`ownerid` are stripped +from the record before it is saved and applied in a second pass once the records +exist. The point is the `IsBatchable` exclusion above: without deferring, every record +that carries one of those attributes falls off the batched path and is imported one at +a time, so a block of inactive or non-default-owner records gets no batching at all. +Deferring keeps those records batchable and moves the state/owner work into a separate +pass that can itself be batched. The end state of each record is unchanged. + +The option is ignored for a block whose records carry *nothing but* state and owner — +a common pattern where the state changes live in their own `Save="UpdateOnly"` block. +Stripping there would leave empty records to save, so `IsStateOwnerOnlyBlock` turns the +option off for the block and logs that it did; `HasAttributesBesidesStateOwner` catches +the same case per record in a mixed block. + +No benchmark numbers are published for this — the gain depends entirely on how many +records in the block carry those attributes. + +### XrmToolBox plugin model + +All three UI classes inherit `PluginControlBase` and implement standard XrmToolBox interfaces (`IMessageBusHost`, `IGitHubPlugin`, `IHelpPlugin`, `IAboutPlugin`). Long-running operations raise events via `ShuffleEventHandler` / `ShuffleEventArgs` rather than blocking the UI thread. + +### Builder controls bind through `Tag` + +Each editor under `XTB/Builder/Controls/` derives from `ControlBase`, which reads and +writes the definition XML purely by convention: every control whose `Tag` is set to +`"AttributeName|required|defaultvalue"` becomes that XML attribute, ordered by +`TabIndex`, and a value equal to the default is omitted from the output. Adding a new +attribute to the schema therefore means adding a control with the matching `Tag` — +there is no separate mapping table. + +`ControlBase` also fills one "Information" box per node from the `` +on the *element* in `ShuffleDefinition.xsd`, via `ShuffleHelper.GetNodeDocumentation`. +It does not resolve documentation on *attributes*, so per-attribute help has to be a +tooltip on the control. + +### Serialization types + +Six types control export format: `Full`, `Simple`, `SimpleWithValue`, `SimpleNoId`, `Explicit`, `Text`. `SimpleWithValue` is the default for Runner. Attributes are sorted alphabetically on export for deterministic diffs. + +### Namespaces + +| Area | Namespace | +|---|---| +| Core logic | `Cinteros.Crm.Utils.Shuffle` | +| Builder UI | `Rappen.XTB.Shuffle.Builder` | +| Runner UI | `Rappen.XTB.Shuffle.Runner` | +| Deployer UI | `Rappen.XTB.ShuffleDeployer` | + +## Key dependencies + +- **XrmToolBoxPackage** — plugin framework (PluginControlBase, connection management) +- **Xrm.Utils.Core** (submodule) — IExecutionContainer, ILogger, entity/service extensions +- **Microsoft.CrmSdk.Workflow** — CRM SDK types +- **System.IO.Compression** — used by Deployer for `.cdpkg` (ZIP) handling diff --git a/README.md b/README.md index bd12acc..f85f19b 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,306 @@ # Xrm.Shuffle for [XrmToolBox](https://www.xrmtoolbox.com/) -### Shuffle Builder 👷‍♀️, Shuffle Runner 🏃 and Shuffle Deployer 🚚 +### Shuffle Builder 🏗️, Shuffle Runner 🏃 and Shuffle Deployer 🚚 *Empower yourself to achieve more.* Created by [@rappen](https://github.com/rappen)
Improving by [@imranakram](https://github.com/imranakram) & [@rappen](https://github.com/rappen) -[XrmToolBox](http://www.xrmtoolbox.com) tools to help compose and run/test **Shuffle Schema Definitions**. +[XrmToolBox](http://www.xrmtoolbox.com) tools to help compose and run/test **Shuffle Schema Definitions** — XML files that define exactly what data and solutions to export or import between Dataverse environments. --- ### *Shuffle tools are now available in the XrmToolBox Tool Library!* 🥳 --- +## The Three Tools + +### 🏗️ Shuffle Builder +The Builder helps you **create and edit Shuffle Definition XML files** through a visual UI — no hand-coding required. + +- Connects to a Dataverse environment to browse entities, attributes, and relationships +- Build `` and `` nodes by pointing and clicking +- Copy, paste, and reorder blocks +- Save `.xml` definition files that are then consumed by the Runner or Deployer +- Available in the XrmToolBox Tool Library as **`Rappen.XrmToolBox.Shuffle.Builder`** + +--- + +### 🏃 Shuffle Runner +The Runner **executes a Shuffle Definition** — exporting data from or importing data into a connected Dataverse environment. + +- Load a definition file and a data file, then hit Run +- Supports both **Export** (Dataverse → XML/CSV file) and **Import** (file → Dataverse) modes +- Multiple serialization styles: Simple, SimpleWithValue, SimpleNoId, Explicit, Text, Full +- Filter records by attribute value or supply your own FetchXML +- High-performance bulk imports using **CreateMultiple/UpdateMultiple** on Dataverse (online) or **ExecuteMultipleRequest** on-premises +- Automatic runtime detection and fallback for maximum compatibility across Dynamics CRM 9.1 and all Dataverse versions +- Generates detailed, timestamped operation logs +- Available in the XrmToolBox Tool Library as **`Rappen.XrmToolBox.Shuffle.Runner`** + +--- + +### 🚚 Shuffle Deployer +The Deployer orchestrates **controlled deployments** of packaged Shuffle definitions across environments. + +- Works with `.cdpkg` / `.cdzip` package files that bundle definition and data files together +- Select which modules within a package to deploy and run them in sequence +- Progress tracking and detailed logs at every step +- Supports **double-click launch**: associate `.cdpkg` files with XrmToolBox and the Deployer will auto-load the package on startup +- Available in the XrmToolBox Tool Library as **`Rappen.XrmToolBox.Shuffle.Deployer`** + +--- + +## Schema Reference + +All three tools are driven by a **Shuffle Definition XML file** that follows the `ShuffleDefinition.xsd` schema. You can author these files in the Builder UI or by hand. Below is a full reference of every element and attribute. + +--- + +### `` — Root element + +| Attribute | Type | Default | Description | +|-----------|------|---------|-------------| +| `Timeout` | int | — | Operation timeout in minutes | +| `StopOnError` | boolean | `false` | Halt all remaining blocks if any block fails | + +Contains a `` child holding any combination of `` and `` elements, processed in order. + +--- + +### `` — Import or export a solution + +| Attribute | Type | Description | +|-----------|------|-------------| +| `Name` | string (required) | Unique name for this block | +| `Path` | string | Folder path to the solution file | +| `File` | string | Explicit solution filename override | + +#### `` (optional child) + +| Attribute | Type | Default | Description | +|-----------|------|---------|-------------| +| `Type` | `Managed` / `Unmanaged` / `Both` / `None` | required | Solution package type to export | +| `SetVersion` | string | — | Override the solution version on export | +| `PublishBeforeExport` | boolean | `false` | Publish all customizations before exporting | +| `TargetVersion` | string | — | Target platform version for the export | + +`` (optional child of ``) — include additional settings components in the export. All boolean, default `false`: + +`AutoNumbering` · `Calendar` · `Customization` · `EmailTracking` · `General` · `Marketing` · `OutlookSync` · `RelationshipRoles` · `IsvConfig` + +#### `` (optional child) + +| Attribute | Type | Default | Description | +|-----------|------|---------|-------------| +| `Type` | `Managed` / `Unmanaged` / `Both` / `None` | required | Expected package type to import | +| `OverwriteSameVersion` | boolean | `true` | Import even when the same version already exists in the target | +| `OverwriteNewerVersion` | boolean | `false` | Import even when the target already has a newer version | +| `ActivateServersideCode` | boolean | required | Activate plug-ins and workflows after import | +| `OverwriteCustomizations` | boolean | required | Overwrite unmanaged customizations | +| `PublishAll` | boolean | required | Publish all after import completes | + +`` — one or more `` elements that must be present in the target before import begins: + +| Attribute | Description | +|-----------|-------------| +| `Name` | Solution unique name | +| `Comparer` | Version comparison rule: `any`, `eq-this`, `ge-this`, `eq`, `ge` | +| `Version` | Required version string (used with `eq` / `ge`) | + +`` — a nested `` element whose blocks are run only after a successful import. + +--- + +### `` — Export or import entity records + +| Attribute | Type | Description | +|-----------|------|-------------| +| `Name` | string (required) | Unique name for this block | +| `Entity` | string (required) | Dataverse entity logical name | +| `Type` | `Entity` / `Intersect` | `Entity` (default) for regular tables; `Intersect` for N:N relationship tables | +| `IntersectName` | string | The intersect entity logical name — required when `Type=Intersect` | + +#### `` (optional child) + +| Attribute | Type | Default | Description | +|-----------|------|---------|-------------| +| `ActiveOnly` | boolean | `false` | Skip inactive/disabled records | + +Choose one of two query modes: + +**Filter-based mode** — combine filters, sorting, and an explicit attribute list: +- `` — add as many as needed +- `` — add as many as needed +- `` containing `` — **required**; defines which fields to include in the export + +**FetchXML mode** — supply your own query (defines both filters and returned attributes): +- `` — paste your raw FetchXML string here; mutually exclusive with the filter-based mode + +#### `` (optional child) + +| Attribute | Type | Default | Description | +|-----------|------|---------|-------------| +| `Save` | `CreateUpdate` / `CreateOnly` / `UpdateOnly` / `Never` | `CreateUpdate` | Whether to create new records, update existing ones, both, or skip saving | +| `Delete` | `None` / `Existing` / `All` | `None` | `None` = no deletes; `Existing` = delete records in target not present in import; `All` = delete all target records first | +| `CreateWithId` | boolean | `false` | Preserve the source record GUID when creating records in the target | +| `UpdateInactive` | boolean | `false` | Allow updating inactive/disabled records | +| `UpdateIdentical` | boolean | `false` | Send an update call even when no field values have changed | +| `BatchSize` | int | `1` | Records per bulk operation batch. Batching is **opt-in**: at the default of `1` every record is imported individually, as it was before batching existed. Set it above `1` to batch — Microsoft recommends ~100 for standard tables. Maximum `1000`. | +| `DeferStateAndOwner` | boolean | `false` | Strip `statecode`, `statuscode`, and `ownerid` from records during import and apply them in a second pass using bulk operations. Records carrying those attributes cannot be batched, so deferring them is what lets such a block use batching at all. | +| `Overwrite` | boolean | — | ⚠️ **Deprecated** — use `Save` instead | + +> **Performance tip:** Shuffle automatically uses **CreateMultiple/UpdateMultiple/UpsertMultiple** bulk operations on Dataverse (online) for maximum throughput, falling back to **ExecuteMultipleRequest** for on-premises CRM 9.1 compatibility, and further falling back to individual operations for CRM 8.x and older. `BatchSize` controls how many records are grouped per API call, and it defaults to `1` — nothing is batched until a definition asks for it. Set it to ~100 to opt in, which is Microsoft's recommendation for standard tables; larger values (up to 1000) may improve throughput for simple operations. Note that `CreateMultiple` and `UpdateMultiple` are a single transaction, so one bad record fails the whole batch, where an unbatched import would have failed only that record. For records with complex plug-ins, keep the value low or leave batching off. + +> **UpsertMultiple optimization:** When importing with `Save="CreateUpdate"`, `CreateWithId="true"`, `UpdateIdentical="true"` and match attributes defined, Shuffle automatically uses **UpsertMultiple** on Dataverse, eliminating the `PreRetrieveAll` queries by letting Dataverse decide whether to create or update each record. No configuration required beyond those attributes — the system detects when Upsert is applicable and uses it automatically. + +> **DeferStateAndOwner optimization:** Records carrying `statecode`, `statuscode` or `ownerid` are excluded from batching, so without this option a block full of inactive or reassigned records is imported one row at a time. With `DeferStateAndOwner="true"` those attributes are stripped before the record is saved, the record goes through the normal batched path, and the state and owner changes are applied afterwards in a second pass. The end state of each record is the same. How much this gains depends on what share of the block carries those attributes — a block where none do gains nothing. Use it when migrating between environments where preserving state and ownership matters. + +> **`StopOnError` and batching:** batching does not change *whether* a run stops on a server error, but it does change what has already happened when it stops. With `StopOnError="true"` batches are sent with `ContinueOnError = false`, so the platform stops at the first faulting record and the remaining records **in that same batch are not executed**. Shuffle names the faulting row, logs how many records in the flight were left undone — `StopOnError: aborting, N record(s) in this batch were not executed` — and aborts the run. Those records are not counted as saved. With `StopOnError="false"` the whole batch is attempted and every faulting row is logged individually as `NNN Update Failed: …` / `NNN Create Failed: …` with the server's own message, and counted in `Failed`, so the closing `Created: … Updated: … Skipped: … Deleted: … Failed: …` line always adds up to the number of rows in the block. Either way, a row the server rejected is never reported as a success. + +#### Import Path Selection + +Shuffle automatically selects the optimal import strategy based on your configuration. Understanding when each path is used helps you configure imports for best performance. + +**Upsert Path** (fastest, Dataverse only) — Used when ALL of these conditions are met: +- `Save="CreateUpdate"` — records may be created or updated +- `CreateWithId="true"` — records include their primary key +- `` has one or more attributes defined +- `Delete="None"` (or not specified) — no deletion of existing records +- `UpdateIdentical="true"` — Upsert never retrieves the existing record, so it cannot tell an identical row from a changed one and always writes. Without this flag the block has asked for identical records to be skipped, which Upsert cannot honour, so the Match-based path is used instead. + +When the Upsert path is active: +- ✅ **UpsertMultiple** sends records directly to Dataverse without pre-querying +- ✅ `PreRetrieveAll` is **automatically bypassed** (not needed since Dataverse decides create vs update) +- ✅ No match queries are executed — Dataverse handles matching internally using the record's primary key +- ⚠️ Falls back gracefully on CRM 9.1 on-premises (ExecuteMultiple + Upsert) or CRM 8.x (individual Create/Update) + +**Match-based Path** (traditional) — Used when ANY of these conditions apply: +- `Save="CreateOnly"` or `Save="UpdateOnly"` — one-directional operations +- `CreateWithId="false"` — records don't include their primary key +- `Delete="Existing"` or `Delete="All"` — deletion requires knowing which records exist +- No `` attributes defined — no way to identify existing records + +When the Match-based path is active: +- 🔍 Each record is matched against target using `` attributes +- 🔍 `PreRetrieveAll="true"` fetches all target records up-front (recommended for large imports on small-to-medium target datasets) +- 🔍 `PreRetrieveAll="false"` (default) queries for matches per-record (better for small imports or very large target datasets) + +| Configuration | Import Path | PreRetrieveAll Effect | +|---------------|-------------|----------------------| +| `Save="CreateUpdate"` + `CreateWithId="true"` + Match defined + `Delete="None"` + `UpdateIdentical="true"` | Upsert | Bypassed (not needed) | +| `Save="CreateUpdate"` + `CreateWithId="false"` | Match-based | Active | +| `UpdateIdentical="false"` (the default) | Match-based | Active | +| `Save="CreateOnly"` (any other flags) | Match-based | Active | +| `Save="UpdateOnly"` (any other flags) | Match-based | Active | +| `Delete="Existing"` or `Delete="All"` | Match-based | Active | +| No `` defined | Direct Create | N/A (no matching) | + +`` — controls how the importer finds existing target records to decide whether to create or update: + +| Attribute | Type | Default | Description | +|-----------|------|---------|-------------| +| `PreRetrieveAll` | boolean | `false` | Fetch all existing target records up-front before import starts. Only applies when using the **Match-based path** (see Import Path Selection above). Recommended for large imports targeting small-to-medium datasets. When the **Upsert path** is active, this flag is automatically bypassed. | + +Add one or more `` children — these are the fields used to match incoming records against existing target records. `Display` is an optional alternate attribute used for the matched value in log output. + +#### `` (optional, repeatable) + +Associates records from another `` — used for N:N relationships or populating lookups: + +| Attribute | Type | Description | +|-----------|------|-------------| +| `Block` | string (required) | Name of the `DataBlock` that provides the related records | +| `Attribute` | string (required) | Lookup attribute on this entity | +| `PK-Attribute` | string | Primary key attribute on the related block (optional; defaults to the block entity's primary key) | +| `IncludeNull` | boolean | Include the relation even when the lookup value is null | + +--- + +## Recent Changes + +### UpsertMultiple bulk operation support +Import operations with `Save="CreateUpdate"` and `CreateWithId="true"` now automatically use **UpsertMultiple** on Dataverse (online), which provides significant performance benefits: + +- **Eliminates PreRetrieveAll queries** — when Upsert is available, the system no longer needs to query existing records to determine create vs. update. Dataverse makes this decision automatically. +- **Single batch for all records** — instead of separate batches for creates and updates, all records go through a unified Upsert batch. +- **Same robust fallback chain** — automatically falls back through multiple tiers: + 1. **UpsertMultiple** (Dataverse online only) + 2. **ExecuteMultipleRequest with UpsertRequest** (CRM 9.1 on-premises) + 3. **Individual UpsertRequest** (fallback) + 4. **Individual Create/Update** (CRM 8.x and older) + +**When Upsert is used automatically:** +- `Save="CreateUpdate"` (not CreateOnly or UpdateOnly) +- `CreateWithId="true"` (records have their source GUID preserved) +- `` attributes are defined +- `Delete` is set to `None` (no deletion of existing records) + +**Example configuration:** +```xml + + + + + + + +``` + +**Performance impact:** For environment-to-environment migrations with `CreateWithId="true"`, imports can be **2-3× faster** because: +1. No `PreRetrieveAll` query overhead +2. No per-record match queries +3. Single unified batch instead of separate create/update batches + +**Backwards compatibility:** Full support maintained for all CRM/Dataverse versions. The system automatically detects capabilities and selects the optimal API path. + +### DeferStateAndOwner optimization for high-performance imports +A new **`DeferStateAndOwner`** attribute on `` enables a two-pass import strategy that dramatically improves performance when importing records with `statecode`, `statuscode`, or `ownerid` attributes: + +- **Pass 1**: Strip state/owner attributes → records become batchable → imported via CreateMultiple/UpdateMultiple +- **Pass 2**: Apply state/owner changes in bulk using UpdateMultiple and batch Assign operations + +**Performance impact**: the gain is proportional to how much of the block was previously unbatchable. On a dataset where nearly every record carried a state or owner attribute, the batchable share went from a few percent to almost all of it; on a block where no record carries them, the option changes nothing. Enabled via: + +```xml + +``` + +This feature is opt-in (default: `false`) to maintain full backwards compatibility. Ideal for environment-to-environment data migrations where preserving record state and ownership is required. + +### CreateMultiple/UpdateMultiple bulk operation support +Import operations now use **CreateMultiple** and **UpdateMultiple** bulk messages on Dataverse (online) for significantly improved performance — up to 2-4× faster than ExecuteMultipleRequest for large datasets. The implementation includes: + +- **Runtime capability detection** — queries `sdkmessagefilter` to detect if bulk operations are supported for each entity type +- **Per-entity caching** — capability checks are cached for the lifetime of the import run +- **Graceful fallback** — automatically falls back to ExecuteMultipleRequest for on-premises CRM 9.1 or entities that don't support bulk operations +- **Full backwards compatibility** — works seamlessly with Dynamics CRM 9.1 on-premises and all Dataverse versions +- **Opt-in** — `BatchSize` defaults to `1`, so existing definitions keep importing record by record until one asks for batching + +Capability detection is automatic — the system works out what the target environment supports and selects the best available API. Batching itself is not: set `BatchSize` above `1` on an Import element to turn it on. + +### Multi-Select OptionSet support +Export and import of Multi-Select OptionSet (OptionSetValueCollection) fields now works correctly. Previously, exported data.xml contained the literal string "OptionSetValueCollection" instead of actual values. + +### ExecuteMultipleRequest batching (legacy) +Import operations on on-premises Dynamics CRM 9.1 use `ExecuteMultipleRequest` for batching (Create, Update, Delete operations). Dataverse (online) environments automatically use the newer and faster CreateMultiple/UpdateMultiple APIs instead. Configurable via the `BatchSize` attribute on the Import element (default: 1, i.e. no batching; max: 1000). The Shuffle Builder UI includes a "Batch size" field and a "Defer state and owner" checkbox on the Import node. + +### Deterministic XML export ordering +Entity attributes are now sorted alphabetically during export, eliminating spurious diffs in version control when re-exporting unchanged data. + +### Bug fixes and performance improvements +- Fixed off-by-one error in CSV/text export that could cause an IndexOutOfRangeException +- Metadata lookups (PrimaryIdAttribute) hoisted out of inner loops to reduce overhead +- Replaced O(n) list searches with HashSet for attribute deduplication during import +- Replaced O(n²) attribute filtering in SelectAttributes with single-pass LINQ approach +- Update failures now log the exception message for easier diagnostics +- A batch that stops on the first fault no longer counts its unexecuted requests as successes; every failed row is logged with its index and fault message +- Records whose lookups point at another record still waiting in the batch are no longer written with the source-system id — the batch is flushed first +- A deferred state or owner change now receives the real id of a record that was created inside a batch, instead of being silently dropped + +--- + ## Home page https://jonasr.app/shuffle/ diff --git a/Rappen.XTB.Shuffle.sln b/Rappen.XTB.Shuffle.sln index ec4433e..43e0783 100644 --- a/Rappen.XTB.Shuffle.sln +++ b/Rappen.XTB.Shuffle.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.5.33516.290 +# Visual Studio Version 18 +VisualStudioVersion = 18.4.11626.88 MinimumVisualStudioVersion = 10.0.40219.1 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Rappen.XTB.Shuffle", "XTB\Rappen.XTB.Shuffle.csproj", "{13AE5564-5C72-4A70-8AC5-00D227E8200A}" EndProject @@ -15,11 +15,26 @@ Project("{D954291E-2A0B-460D-934E-DC6B0785DB48}") = "Xrm.Utils.Core.Common", "Xr EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{5DE162B2-F4D4-4D6E-BBA7-5F8DB8A14358}" ProjectSection(SolutionItems) = preProject + .gitignore = .gitignore + LICENCE = LICENCE + README.md = README.md XTB\ShuffleBuilder.nuspec = XTB\ShuffleBuilder.nuspec XTB\ShuffleDeployer.nuspec = XTB\ShuffleDeployer.nuspec XTB\ShuffleRunner.nuspec = XTB\ShuffleRunner.nuspec EndProjectSection EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".github", ".github", "{D9752343-576F-48AD-A576-4CFC3499C7F4}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "workflows", "workflows", "{7F7BC1BA-C6C6-47B9-8D60-268822E0A334}" + ProjectSection(SolutionItems) = preProject + .github\workflows\build.yml = .github\workflows\build.yml + .github\workflows\release.yml = .github\workflows\release.yml + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{3F6B08D4-9C1A-4E77-A5B2-8E93D1C60F52}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Xrm.Shuffle.Core.Tests", "tests\Xrm.Shuffle.Core.Tests\Xrm.Shuffle.Core.Tests.csproj", "{7C3E9A21-4B8D-4E52-9F1C-6D0A5B2E4417}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -30,6 +45,10 @@ Global {13AE5564-5C72-4A70-8AC5-00D227E8200A}.Debug|Any CPU.Build.0 = Debug|Any CPU {13AE5564-5C72-4A70-8AC5-00D227E8200A}.Release|Any CPU.ActiveCfg = Release|Any CPU {13AE5564-5C72-4A70-8AC5-00D227E8200A}.Release|Any CPU.Build.0 = Release|Any CPU + {7C3E9A21-4B8D-4E52-9F1C-6D0A5B2E4417}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7C3E9A21-4B8D-4E52-9F1C-6D0A5B2E4417}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7C3E9A21-4B8D-4E52-9F1C-6D0A5B2E4417}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7C3E9A21-4B8D-4E52-9F1C-6D0A5B2E4417}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -37,6 +56,8 @@ Global GlobalSection(NestedProjects) = preSolution {5C10BFE2-AFAA-4B01-A570-B30EF41DE1F0} = {44DE50B5-DA2B-4BD6-9D10-8BB345F68226} {A939CF3B-672A-4F68-8E6A-89EFE8C8CFBB} = {17CCEFED-E37B-47CF-BD9F-E7596E65FCE9} + {7F7BC1BA-C6C6-47B9-8D60-268822E0A334} = {D9752343-576F-48AD-A576-4CFC3499C7F4} + {7C3E9A21-4B8D-4E52-9F1C-6D0A5B2E4417} = {3F6B08D4-9C1A-4E77-A5B2-8E93D1C60F52} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {ED295346-E5B5-4006-855E-1100CEB0F456} @@ -46,5 +67,7 @@ Global Xrm.Utils.Core\Xrm.Utils.Core.Common\Xrm.Utils.Core.Common.projitems*{13ae5564-5c72-4a70-8ac5-00d227e8200a}*SharedItemsImports = 4 shared\Xrm.Shuffle.Core\Xrm.Shuffle.Core.projitems*{5c10bfe2-afaa-4b01-a570-b30ef41de1f0}*SharedItemsImports = 13 Xrm.Utils.Core\Xrm.Utils.Core.Common\Xrm.Utils.Core.Common.projitems*{a939cf3b-672a-4f68-8e6a-89efe8c8cfbb}*SharedItemsImports = 13 + shared\Xrm.Shuffle.Core\Xrm.Shuffle.Core.projitems*{7c3e9a21-4b8d-4e52-9f1c-6d0a5b2e4417}*SharedItemsImports = 4 + Xrm.Utils.Core\Xrm.Utils.Core.Common\Xrm.Utils.Core.Common.projitems*{7c3e9a21-4b8d-4e52-9f1c-6d0a5b2e4417}*SharedItemsImports = 4 EndGlobalSection EndGlobal diff --git a/XTB/Builder/Controls/DataBlockImportControl.Designer.cs b/XTB/Builder/Controls/DataBlockImportControl.Designer.cs index cf6455c..232de5c 100644 --- a/XTB/Builder/Controls/DataBlockImportControl.Designer.cs +++ b/XTB/Builder/Controls/DataBlockImportControl.Designer.cs @@ -41,6 +41,13 @@ private void InitializeComponent() this.txtOverwrite = new System.Windows.Forms.TextBox(); this.label7 = new System.Windows.Forms.Label(); this.chkUpdateIdentical = new System.Windows.Forms.CheckBox(); + this.label8 = new System.Windows.Forms.Label(); + this.txtBatchSize = new System.Windows.Forms.TextBox(); + this.label9 = new System.Windows.Forms.Label(); + this.chkDeferStateAndOwner = new System.Windows.Forms.CheckBox(); + this.lblDeferStateAndOwnerHelp = new System.Windows.Forms.Label(); + this.components = new System.ComponentModel.Container(); + this.tooltips = new System.Windows.Forms.ToolTip(this.components); this.SuspendLayout(); // // chkCreateWithId @@ -77,7 +84,7 @@ private void InitializeComponent() "Never"}); this.cmbSave.Location = new System.Drawing.Point(213, 31); this.cmbSave.Name = "cmbSave"; - this.cmbSave.Size = new System.Drawing.Size(234, 21); + this.cmbSave.Size = new System.Drawing.Size(236, 21); this.cmbSave.TabIndex = 6; this.cmbSave.Tag = "Save|false|CreateUpdate"; // @@ -113,7 +120,7 @@ private void InitializeComponent() "All"}); this.cmbDelete.Location = new System.Drawing.Point(213, 58); this.cmbDelete.Name = "cmbDelete"; - this.cmbDelete.Size = new System.Drawing.Size(234, 21); + this.cmbDelete.Size = new System.Drawing.Size(236, 21); this.cmbDelete.TabIndex = 8; this.cmbDelete.Tag = "Delete|false|None"; // @@ -139,7 +146,7 @@ private void InitializeComponent() // lblDeprecated // this.lblDeprecated.AutoSize = true; - this.lblDeprecated.Location = new System.Drawing.Point(4, 154); + this.lblDeprecated.Location = new System.Drawing.Point(4, 185); this.lblDeprecated.Name = "lblDeprecated"; this.lblDeprecated.Size = new System.Drawing.Size(66, 13); this.lblDeprecated.TabIndex = 12; @@ -149,7 +156,7 @@ private void InitializeComponent() // lblDeprOverwrite // this.lblDeprOverwrite.AutoSize = true; - this.lblDeprOverwrite.Location = new System.Drawing.Point(7, 171); + this.lblDeprOverwrite.Location = new System.Drawing.Point(7, 202); this.lblDeprOverwrite.Name = "lblDeprOverwrite"; this.lblDeprOverwrite.Size = new System.Drawing.Size(52, 13); this.lblDeprOverwrite.TabIndex = 13; @@ -158,7 +165,7 @@ private void InitializeComponent() // // txtOverwrite // - this.txtOverwrite.Location = new System.Drawing.Point(213, 168); + this.txtOverwrite.Location = new System.Drawing.Point(213, 199); this.txtOverwrite.Name = "txtOverwrite"; this.txtOverwrite.Size = new System.Drawing.Size(234, 20); this.txtOverwrite.TabIndex = 20; @@ -184,10 +191,73 @@ private void InitializeComponent() this.chkUpdateIdentical.Tag = "UpdateIdentical|false|false"; this.chkUpdateIdentical.UseVisualStyleBackColor = true; // - // DataBlockImportControl + // label8 + // + this.label8.AutoSize = true; + this.label8.Location = new System.Drawing.Point(4, 138); + this.label8.Name = "label8"; + this.label8.Size = new System.Drawing.Size(56, 13); + this.label8.TabIndex = 21; + this.label8.Text = "Batch size"; // + // txtBatchSize + // + this.txtBatchSize.Location = new System.Drawing.Point(213, 135); + this.txtBatchSize.Name = "txtBatchSize"; + this.txtBatchSize.Size = new System.Drawing.Size(80, 20); + this.txtBatchSize.TabIndex = 22; + this.txtBatchSize.Tag = "BatchSize|false|1"; + this.tooltips.SetToolTip(this.txtBatchSize, "Number of records sent to the server per bulk request. Batching is off by default (1). Set above 1 to enable it - note that CreateMultiple and UpdateMultiple are transactional, so one bad record fails the whole batch. Max 1000."); + // + // label9 + // + this.label9.AutoSize = true; + this.label9.Location = new System.Drawing.Point(4, 161); + this.label9.Name = "label9"; + this.label9.Size = new System.Drawing.Size(114, 13); + this.label9.TabIndex = 23; + this.label9.Text = "Defer state and owner"; + this.tooltips.SetToolTip(this.label9, DeferStateAndOwnerHelp); + // + // chkDeferStateAndOwner + // + this.chkDeferStateAndOwner.AutoSize = true; + this.chkDeferStateAndOwner.Location = new System.Drawing.Point(213, 161); + this.chkDeferStateAndOwner.Name = "chkDeferStateAndOwner"; + this.chkDeferStateAndOwner.Size = new System.Drawing.Size(15, 14); + this.chkDeferStateAndOwner.TabIndex = 24; + this.chkDeferStateAndOwner.Tag = "DeferStateAndOwner|false|false"; + this.chkDeferStateAndOwner.UseVisualStyleBackColor = true; + this.tooltips.SetToolTip(this.chkDeferStateAndOwner, DeferStateAndOwnerHelp); + // + // lblDeferStateAndOwnerHelp + // + this.lblDeferStateAndOwnerHelp.AutoSize = true; + this.lblDeferStateAndOwnerHelp.Cursor = System.Windows.Forms.Cursors.Help; + this.lblDeferStateAndOwnerHelp.Font = new System.Drawing.Font("Microsoft Sans Serif", 8.25F, System.Drawing.FontStyle.Bold); + this.lblDeferStateAndOwnerHelp.ForeColor = System.Drawing.SystemColors.HotTrack; + this.lblDeferStateAndOwnerHelp.Location = new System.Drawing.Point(234, 160); + this.lblDeferStateAndOwnerHelp.Name = "lblDeferStateAndOwnerHelp"; + this.lblDeferStateAndOwnerHelp.Size = new System.Drawing.Size(13, 13); + this.lblDeferStateAndOwnerHelp.TabIndex = 25; + this.lblDeferStateAndOwnerHelp.Text = "?"; + this.tooltips.SetToolTip(this.lblDeferStateAndOwnerHelp, DeferStateAndOwnerHelp); + // + // tooltips + // + this.tooltips.AutoPopDelay = 20000; + this.tooltips.InitialDelay = 300; + this.tooltips.ReshowDelay = 100; + // + // DataBlockImportControl + // this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F); this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; + this.Controls.Add(this.lblDeferStateAndOwnerHelp); + this.Controls.Add(this.label9); + this.Controls.Add(this.chkDeferStateAndOwner); + this.Controls.Add(this.label8); + this.Controls.Add(this.txtBatchSize); this.Controls.Add(this.label7); this.Controls.Add(this.chkUpdateIdentical); this.Controls.Add(this.txtOverwrite); @@ -202,7 +272,25 @@ private void InitializeComponent() this.Controls.Add(this.label1); this.Controls.Add(this.chkCreateWithId); this.Name = "DataBlockImportControl"; - this.Size = new System.Drawing.Size(450, 205); + this.Size = new System.Drawing.Size(452, 258); + this.Controls.SetChildIndex(this.chkCreateWithId, 0); + this.Controls.SetChildIndex(this.label1, 0); + this.Controls.SetChildIndex(this.cmbSave, 0); + this.Controls.SetChildIndex(this.label2, 0); + this.Controls.SetChildIndex(this.cmbDelete, 0); + this.Controls.SetChildIndex(this.label3, 0); + this.Controls.SetChildIndex(this.chkUpdateInactive, 0); + this.Controls.SetChildIndex(this.label4, 0); + this.Controls.SetChildIndex(this.lblDeprecated, 0); + this.Controls.SetChildIndex(this.lblDeprOverwrite, 0); + this.Controls.SetChildIndex(this.txtOverwrite, 0); + this.Controls.SetChildIndex(this.chkUpdateIdentical, 0); + this.Controls.SetChildIndex(this.label7, 0); + this.Controls.SetChildIndex(this.txtBatchSize, 0); + this.Controls.SetChildIndex(this.label8, 0); + this.Controls.SetChildIndex(this.chkDeferStateAndOwner, 0); + this.Controls.SetChildIndex(this.label9, 0); + this.Controls.SetChildIndex(this.lblDeferStateAndOwnerHelp, 0); this.ResumeLayout(false); this.PerformLayout(); @@ -223,5 +311,11 @@ private void InitializeComponent() private System.Windows.Forms.TextBox txtOverwrite; private System.Windows.Forms.Label label7; private System.Windows.Forms.CheckBox chkUpdateIdentical; + private System.Windows.Forms.Label label8; + private System.Windows.Forms.TextBox txtBatchSize; + private System.Windows.Forms.Label label9; + private System.Windows.Forms.CheckBox chkDeferStateAndOwner; + private System.Windows.Forms.Label lblDeferStateAndOwnerHelp; + private System.Windows.Forms.ToolTip tooltips; } } diff --git a/XTB/Builder/Controls/DataBlockImportControl.cs b/XTB/Builder/Controls/DataBlockImportControl.cs index 685d970..d87d9d7 100644 --- a/XTB/Builder/Controls/DataBlockImportControl.cs +++ b/XTB/Builder/Controls/DataBlockImportControl.cs @@ -5,6 +5,16 @@ namespace Rappen.XTB.Shuffle.Builder.Controls { public partial class DataBlockImportControl : ControlBase { + private const string DeferStateAndOwnerHelp = + "Defer state and owner:\r\n" + + "Records that carry statecode, statuscode or ownerid cannot be sent in a bulk request, so they " + + "are imported one at a time. With this option those attributes are stripped off before the " + + "record is saved and applied afterwards in a second pass, which keeps the records themselves " + + "on the batched path.\r\n" + + "Benefit: noticeably faster import of blocks where most or all records are inactive or owned by " + + "someone other than the importing user. The end result is the same - every record still ends up " + + "with its intended state and owner."; + public DataBlockImportControl(Dictionary collection, ShuffleBuilder shuffleBuilder) : base(collection, shuffleBuilder) { diff --git a/XTB/Rappen.XTB.Shuffle.csproj b/XTB/Rappen.XTB.Shuffle.csproj index ca856dd..77f626e 100644 --- a/XTB/Rappen.XTB.Shuffle.csproj +++ b/XTB/Rappen.XTB.Shuffle.csproj @@ -347,7 +347,7 @@ 13.0.4 - 7.3.0 + 7.9.0 4.3.0 diff --git a/XTB/ShuffleBuilder.nuspec b/XTB/ShuffleBuilder.nuspec index d7892fc..2f6cece 100644 --- a/XTB/ShuffleBuilder.nuspec +++ b/XTB/ShuffleBuilder.nuspec @@ -5,7 +5,7 @@ 1.0.0 Shuffle Builder for XrmToolBox Jonas Rapp, Imran Akram - rappen + rappen, imranakram https://jonasr.app/shuffle/ https://jonasr.app/wp-content/uploads/Shuffle-2B.png false @@ -19,9 +19,15 @@ Build schema files for the Shuffle. Empower yourself to achieve more. - Updated dependencies and bug fixes. +- New Batch size field on Import configuration, opting a block in to bulk-message batching (default 1, no batching) +- New Defer state and owner checkbox on Import configuration, with a tooltip explaining when it helps +- Multi-Select OptionSet (OptionSetValueCollection) export/import support +- Deterministic attribute ordering in XML export for clean version control diffs +- Fixed off-by-one error in CSV/text export +- Performance improvements: metadata lookup hoisting, HashSet deduplication, optimized attribute filtering +- Update failure messages now include exception details - Copyright 2023-2025 Jonas Rapp, Imran Akram + Copyright 2023-2026 Jonas Rapp, Imran Akram XrmToolBox Shuffle @@ -29,6 +35,5 @@ - \ No newline at end of file diff --git a/XTB/ShuffleDeployer.nuspec b/XTB/ShuffleDeployer.nuspec index 715c571..bc4d17c 100644 --- a/XTB/ShuffleDeployer.nuspec +++ b/XTB/ShuffleDeployer.nuspec @@ -5,7 +5,7 @@ 1.0.0 Shuffle Deployer for XrmToolBox Jonas Rapp, Imran Akram - rappen + rappen, imranakram https://jonasr.app/shuffle/ https://jonasr.app/wp-content/uploads/Shuffle2-D.png false @@ -19,7 +19,16 @@ Deploy solutions and datas with the Shuffle. Empower yourself to achieve more. - Updated dependencies and bug fixes. +- Opt-in batching for import operations: set BatchSize above 1 to enable it (default 1, no batching) +- New Import option DeferStateAndOwner: statecode/statuscode/ownerid are applied in a second pass so the records themselves stay batchable +- Batching now also engages for blocks that use Match, except where a live match query has to see the pending records first +- Failed rows in a batch are reported and no longer counted as successes +- Rows rejected because they match several records in the target now log the row number, like every other result line +- Multi-Select OptionSet (OptionSetValueCollection) export/import support +- Deterministic attribute ordering in XML export for clean version control diffs +- Fixed off-by-one error in CSV/text export +- Performance improvements: metadata lookup hoisting, HashSet deduplication, optimized attribute filtering +- Update failure messages now include exception details Copyright 2023-2026 Jonas Rapp, Imran Akram XrmToolBox Shuffle @@ -29,6 +38,5 @@ - \ No newline at end of file diff --git a/XTB/ShuffleRunner.nuspec b/XTB/ShuffleRunner.nuspec index 7f24533..aa4df2b 100644 --- a/XTB/ShuffleRunner.nuspec +++ b/XTB/ShuffleRunner.nuspec @@ -5,7 +5,7 @@ 1.0.0 Shuffle Runner for XrmToolBox Jonas Rapp, Imran Akram - rappen + rappen, imranakram https://jonasr.app/shuffle/ https://jonasr.app/wp-content/uploads/Shuffle2-R.png false @@ -19,7 +19,16 @@ Export and Import with the Shuffle. Empower yourself to achieve more. - Updated dependencies and bug fixes. +- Opt-in batching for import operations: set BatchSize above 1 to enable it (default 1, no batching) +- New Import option DeferStateAndOwner: statecode/statuscode/ownerid are applied in a second pass so the records themselves stay batchable +- Batching now also engages for blocks that use Match, except where a live match query has to see the pending records first +- Failed rows in a batch are reported and no longer counted as successes +- Rows rejected because they match several records in the target now log the row number, like every other result line +- Multi-Select OptionSet (OptionSetValueCollection) export/import support +- Deterministic attribute ordering in XML export for clean version control diffs +- Fixed off-by-one error in CSV/text export +- Performance improvements: metadata lookup hoisting, HashSet deduplication, optimized attribute filtering +- Update failure messages now include exception details Copyright 2023-2026 Jonas Rapp, Imran Akram XrmToolBox Shuffle @@ -29,6 +38,5 @@ - \ No newline at end of file diff --git a/shared/Xrm.Shuffle.Core/Resources/ShuffleDefinition.cs b/shared/Xrm.Shuffle.Core/Resources/ShuffleDefinition.cs index cd079d4..1cf9d07 100644 --- a/shared/Xrm.Shuffle.Core/Resources/ShuffleDefinition.cs +++ b/shared/Xrm.Shuffle.Core/Resources/ShuffleDefinition.cs @@ -87,17 +87,29 @@ public partial class DataBlockImport { /// [System.Xml.Serialization.XmlAttributeAttribute()] public bool Overwrite; - + /// [System.Xml.Serialization.XmlIgnoreAttribute()] public bool OverwriteSpecified; - + + /// Number of records per CreateMultiple/UpdateMultiple batch. Batching is off by default; set this above 1 to enable it. Max 1000. Microsoft recommends ~100 for standard tables. + [System.Xml.Serialization.XmlAttributeAttribute()] + [System.ComponentModel.DefaultValueAttribute(1)] + public int BatchSize; + + /// Strip statecode/statuscode/ownerid from records during import and apply them in a second pass using bulk operations. Significantly improves performance for datasets with state/owner attributes. Default: false. + [System.Xml.Serialization.XmlAttributeAttribute()] + [System.ComponentModel.DefaultValueAttribute(false)] + public bool DeferStateAndOwner; + public DataBlockImport() { this.CreateWithId = false; this.Save = SaveTypes.CreateUpdate; this.Delete = DeleteTypes.None; this.UpdateInactive = false; this.UpdateIdentical = false; + this.BatchSize = 1; + this.DeferStateAndOwner = false; } } diff --git a/shared/Xrm.Shuffle.Core/Resources/ShuffleDefinition.xsd b/shared/Xrm.Shuffle.Core/Resources/ShuffleDefinition.xsd index dc5d04d..1bf7b9b 100644 --- a/shared/Xrm.Shuffle.Core/Resources/ShuffleDefinition.xsd +++ b/shared/Xrm.Shuffle.Core/Resources/ShuffleDefinition.xsd @@ -263,7 +263,16 @@ DEPRECATED. Use Save attribute instead. - + + + Number of records per CreateMultiple/UpdateMultiple batch, falling back to ExecuteMultipleRequest. Batching is off by default; set this above 1 to enable it. Max 1000. + + + + + Strip statecode/statuscode/ownerid from records during import and apply them in a second pass, keeping the records themselves batchable. + + diff --git a/shared/Xrm.Shuffle.Core/Shuffle-LCG-configuration.xml b/shared/Xrm.Shuffle.Core/Shuffle-LCG-configuration.xml index 7a2f715..37b2391 100644 --- a/shared/Xrm.Shuffle.Core/Shuffle-LCG-configuration.xml +++ b/shared/Xrm.Shuffle.Core/Shuffle-LCG-configuration.xml @@ -1,7 +1,7 @@ 1.2020.2.1 - C:\Dev\GitHub\Shuffle\Innofactor.Crm.CI\shared\Rappen.XTB.Shuffle + .\shared\Xrm.Shuffle.Core Cinteros.Crm.Utils.Shuffle true Const diff --git a/shared/Xrm.Shuffle.Core/ShuffleDataExport.cs b/shared/Xrm.Shuffle.Core/ShuffleDataExport.cs index 01afcc1..2def96d 100644 --- a/shared/Xrm.Shuffle.Core/ShuffleDataExport.cs +++ b/shared/Xrm.Shuffle.Core/ShuffleDataExport.cs @@ -103,20 +103,13 @@ private static void SelectAttributes(IExecutionContainer container, EntityCollec { foreach (var entity in cExportEntities.Entities) { - var i = 0; - var x = new List(entity.Attributes.Keys); - while (i < entity.Attributes.Count) + var primaryIdAttribute = container.Entity(entity.LogicalName).PrimaryIdAttribute; + var keysToRemove = entity.Attributes.Keys + .Where(attr => attr != primaryIdAttribute && !IncludeAttribute(attr, lAttributes)) + .ToList(); + foreach (var key in keysToRemove) { - var attr = x[i]; - if (attr != container.Entity(entity.LogicalName).PrimaryIdAttribute && !IncludeAttribute(attr, lAttributes)) - { - entity.Attributes.Remove(attr); - x.Remove(attr); - } - else - { - i++; - } + entity.Attributes.Remove(key); } foreach (var nullattribute in lNullAttributes) { diff --git a/shared/Xrm.Shuffle.Core/ShuffleDataImport.cs b/shared/Xrm.Shuffle.Core/ShuffleDataImport.cs index 547648b..718c84d 100644 --- a/shared/Xrm.Shuffle.Core/ShuffleDataImport.cs +++ b/shared/Xrm.Shuffle.Core/ShuffleDataImport.cs @@ -6,6 +6,7 @@ using global::Xrm.Utils.Core.Common.Misc; using Microsoft.Crm.Sdk.Messages; using Microsoft.Xrm.Sdk; + using Microsoft.Xrm.Sdk.Messages; using Microsoft.Xrm.Sdk.Query; using System; using System.Collections.Generic; @@ -16,6 +17,62 @@ public partial class Shuffler { + #region Bulk Operation Support Cache + + /// + /// Cache for CreateMultiple support per entity logical name. + /// True = supported, False = not supported, null = not yet checked. + /// + private Dictionary createMultipleSupportCache = new Dictionary(); + + /// + /// Cache for UpdateMultiple support per entity logical name. + /// True = supported, False = not supported, null = not yet checked. + /// + private Dictionary updateMultipleSupportCache = new Dictionary(); + + /// + /// Cache for UpsertMultiple support per entity logical name. + /// True = supported, False = not supported, null = not yet checked. + /// + private Dictionary upsertMultipleSupportCache = new Dictionary(); + + /// + /// Cache for Upsert (single) support per entity logical name. + /// True = supported, False = not supported, null = not yet checked. + /// + private Dictionary upsertSupportCache = new Dictionary(); + + /// + /// Fault code indicating the message is not implemented (used for on-premises fallback detection). + /// + private const int MessageNotImplementedErrorCode = unchecked((int)0x80040265); + + /// + /// Deferred state changes for the block currently being imported. + /// Held as a field so that the batch flush methods can fill in the actual record id, + /// which is only known once the batch has been sent. + /// + private List deferredStates = new List(); + + /// + /// Deferred owner changes for the block currently being imported. See . + /// + private List deferredOwners = new List(); + + /// + /// Names the record that actually faulted inside a batch flush, so the per-record catch + /// can label the error with it. + /// + /// + /// A flush is triggered by whichever record fills the batch, so the enclosing loop + /// variable points at the last record enqueued - not at the one that failed several + /// records earlier. Null when the fault did not come from a batch. + /// + private string batchFailureLabel; + + #endregion Bulk Operation Support Cache + #region Private Methods private static bool EntityAttributesEqual(IExecutionContainer container, List matchattributes, Entity entity1, Entity entity2) @@ -55,7 +112,14 @@ private static string GetEntityDisplayString(IExecutionContainer container, Data matchdisplay = attribute.Name; } var matchvalue = ""; - if (cdEntity.Contains(matchdisplay, true)) + if (matchdisplay == container.Entity(cdEntity.LogicalName).PrimaryIdAttribute) + { // The primary key is carried in Entity.Id, never in Entity.Attributes, so + // the Contains check below can never see it and every block matching on the + // primary key would log for every record. EntityAttributesEqual + // special-cases it the same way when comparing. + matchvalue = cdEntity.Id.ToString(); + } + else if (cdEntity.Contains(matchdisplay, true)) { if (cdEntity[matchdisplay] is EntityReference) { // Don't use PropertyAsString, that would perform GetRelated that we don't want due to performance @@ -71,7 +135,12 @@ private static string GetEntityDisplayString(IExecutionContainer container, Data } else { - matchvalue = container.Attribute(matchdisplay).On(cdEntity).ToString(); + // Records deserialized from a data file carry no FormattedValues, so the + // fallback has to unwrap the SDK type itself - a plain ToString() on an + // OptionSetValue or Money renders the type name, not the value. + matchvalue = cdEntity.FormattedValues.Contains(matchdisplay) + ? cdEntity.FormattedValues[matchdisplay] + : container.AttributeAsBaseType(cdEntity, matchdisplay, string.Empty, true)?.ToString() ?? ""; } } unique.Add(matchvalue); @@ -242,18 +311,15 @@ private EntityCollection GetMatchingRecordsFromPreRetrieved(IExecutionContainer private List GetUpdateAttributes(EntityCollection entities) { - var result = new List(); + var result = new HashSet(); foreach (var entity in entities.Entities) { foreach (var attribute in entity.Attributes.Keys) { - if (!result.Contains(attribute)) - { - result.Add(attribute); - } + result.Add(attribute); } } - return result; + return result.ToList(); } private Tuple ImportDataBlock(IExecutionContainer container, DataBlock block, EntityCollection cEntities) @@ -286,6 +352,41 @@ private Tuple ImportDataBloc var matchattributes = GetMatchAttributes(block.Import.Match); var updateattributes = !updateidentical ? GetUpdateAttributes(cEntities) : new List(); var preretrieveall = block.Import.Match?.PreRetrieveAll == true; + var batchsize = Math.Max(1, Math.Min(block.Import.BatchSize, 1000)); + var deferStateAndOwner = block.Import.DeferStateAndOwner; + + if (deferStateAndOwner && IsStateOwnerOnlyBlock(cEntities)) + { + // A block that carries nothing but state and owner is already a second pass in + // its own right - a common pattern where the state changes live in a separate + // UpdateOnly block. Deferring here would strip every attribute and leave empty + // records to save, so the option is ignored rather than obeyed. + deferStateAndOwner = false; + SendLine(container, "DeferStateAndOwner ignored - this block carries no attributes besides state and owner"); + } + + if (deferStateAndOwner) + { + SendLine(container, "DeferStateAndOwner enabled - state/owner will be applied in second pass"); + } + + // Determine if we can use Upsert path (eliminates need for PreRetrieveAll queries) + // Upsert is optimal when: Save=CreateUpdate, records have ID (CreateWithId), records are batchable, + // AND UpdateIdentical=true (because Upsert cannot skip identical records - we don't retrieve existing data to compare) + var canUseUpsert = save == SaveTypes.CreateUpdate && + includeid && + matchattributes.Count > 0 && + delete == DeleteTypes.None && + updateidentical; + + if (canUseUpsert) + { + SendLine(container, "Upsert path enabled - records will be upserted without pre-retrieval queries"); + if (preretrieveall) + { + SendLine(container, "Note: PreRetrieveAll is not needed when using Upsert and will be skipped"); + } + } SendLine(container); SendLine(container, $"Importing block {name} - {cEntities.Count()} records "); @@ -299,47 +400,56 @@ private Tuple ImportDataBloc qDelete.ColumnSet.AddColumn(container.Entity(entity).PrimaryNameAttribute); var deleterecords = container.RetrieveMultiple(qDelete); - //var deleterecords = Entity.RetrieveMultiple(crmsvc, qDelete, log); SendLine(container, $"Deleting ALL {entity} - {deleterecords.Count()} records"); + var deleteBatch = new List(); foreach (var record in deleterecords.Entities) { SendLine(container, "{0:000} Deleting existing: {1}", i, record); - try + deleteBatch.Add(record); + if (deleteBatch.Count >= batchsize) { - container.Delete(record); - deleted++; - } - catch (FaultException ex) - { - if (ex.Message.ToUpperInvariant().Contains("DOES NOT EXIST")) - { // This may happen through delayed cascade delete in CRM - SendLine(container, " ...already deleted"); - } - else - { - throw; - } + FlushPendingDeletes(container, deleteBatch, ref deleted, ref failed); } i++; } + FlushPendingDeletes(container, deleteBatch, ref deleted, ref failed); } var totalRecords = cEntities.Count(); i = 1; EntityCollection cAllRecordsToMatch = null; + var pendingCreates = new List(); + var pendingUpdates = new List(); + var pendingUpserts = new List(); + deferredStates = new List(); + deferredOwners = new List(); foreach (var cdEntity in cEntities.Entities) { var unique = cdEntity.Id.ToString(); + batchFailureLabel = null; SendStatus(-1, -1, totalRecords, i); try { var oldid = cdEntity.Id; var newid = Guid.Empty; + // ReplaceGuids rewrites this record's lookups using guidmap, so any record it + // points at must already be committed. Flush first if this record references + // one that is still pending, otherwise the lookup keeps the source-system id. + if (ReferencesPendingCreate(cdEntity, pendingCreates)) + { + FlushPendingCreates(container, pendingCreates, ref created, ref failed, references); + } + ReplaceGuids(container, cdEntity, includeid); ReplaceUpdateInfo(cdEntity); unique = GetEntityDisplayString(container, block.Import.Match, cdEntity); SendStatus(null, unique); + if (deferStateAndOwner) + { + StripAndDeferStateOwner(cdEntity, deferredStates, deferredOwners, i, unique); + } + if (!block.TypeSpecified || block.Type == EntityTypes.Entity) { #region Entity @@ -357,19 +467,69 @@ private Tuple ImportDataBloc { cdEntity.Id = Guid.Empty; } - if (SaveEntity(container, cdEntity, null, updateinactive, updateidentical, i, unique)) + if (IsBatchable(cdEntity)) + { + pendingCreates.Add(new PendingCreate { Entity = cdEntity, OldId = oldid, Position = i, Identifier = unique }); + if (pendingCreates.Count >= batchsize) + { + FlushPendingCreates(container, pendingCreates, ref created, ref failed, references); + } + } + else { - created++; - newid = cdEntity.Id; - references.Add(cdEntity.ToEntityReference()); + FlushPendingCreates(container, pendingCreates, ref created, ref failed, references); + FlushPendingUpdates(container, pendingUpdates, ref updated, ref failed, references); + if (SaveEntity(container, cdEntity, null, updateinactive, updateidentical, i, unique)) + { + created++; + newid = cdEntity.Id; + references.Add(cdEntity.ToEntityReference()); + } } } } + else if (canUseUpsert && IsBatchable(cdEntity)) + { + // Upsert path: skip match queries entirely, let Dataverse decide create vs update + pendingUpserts.Add(new PendingUpsert { Entity = cdEntity, OldId = oldid, Position = i, Identifier = unique }); + if (pendingUpserts.Count >= batchsize) + { + FlushPendingUpserts(container, pendingUpserts, ref created, ref updated, ref failed, references); + } + newid = cdEntity.Id; + } + else if (canUseUpsert && !IsBatchable(cdEntity)) + { + // Non-batchable record in Upsert mode: flush batches and use SaveEntity + FlushPendingUpserts(container, pendingUpserts, ref created, ref updated, ref failed, references); + FlushPendingCreates(container, pendingCreates, ref created, ref failed, references); + FlushPendingUpdates(container, pendingUpdates, ref updated, ref failed, references); + if (SaveEntity(container, cdEntity, null, updateinactive, updateidentical, i, unique)) + { + // SaveEntity handles create vs update detection internally when match is null + // Since we're in upsert mode with includeid=true, the record has an ID + // We count this as "updated" since we don't know if it was created or updated + updated++; + newid = cdEntity.Id; + references.Add(cdEntity.ToEntityReference()); + } + } else { + // Original match-based path. + // A live match query must see the records created so far, so the batch + // has to be flushed first. PreRetrieveAll matches against a snapshot + // taken once at the start of the block, which never sees records created + // during the block whether we flush or not - so there the flush buys + // nothing and would defeat batching for every matched block. + if (!preretrieveall && pendingCreates.Count > 0) + { + FlushPendingCreates(container, pendingCreates, ref created, ref failed, references); + } var matches = GetMatchingRecords(container, cdEntity, matchattributes, updateattributes, preretrieveall, ref cAllRecordsToMatch); if (delete == DeleteTypes.All || (matches.Count() == 1 && delete == DeleteTypes.Existing)) { + FlushPendingUpdates(container, pendingUpdates, ref updated, ref failed, references); foreach (var cdMatch in matches.Entities) { SendLine(container, "{0:000} Deleting existing: {1}", i, unique); @@ -405,11 +565,24 @@ private Tuple ImportDataBloc { cdEntity.Id = Guid.Empty; } - if (SaveEntity(container, cdEntity, null, updateinactive, updateidentical, i, unique)) + if (IsBatchable(cdEntity)) { - created++; - newid = cdEntity.Id; - references.Add(cdEntity.ToEntityReference()); + pendingCreates.Add(new PendingCreate { Entity = cdEntity, OldId = oldid, Position = i, Identifier = unique }); + if (pendingCreates.Count >= batchsize) + { + FlushPendingCreates(container, pendingCreates, ref created, ref failed, references); + } + } + else + { + FlushPendingCreates(container, pendingCreates, ref created, ref failed, references); + FlushPendingUpdates(container, pendingUpdates, ref updated, ref failed, references); + if (SaveEntity(container, cdEntity, null, updateinactive, updateidentical, i, unique)) + { + created++; + newid = cdEntity.Id; + references.Add(cdEntity.ToEntityReference()); + } } } } @@ -419,14 +592,42 @@ private Tuple ImportDataBloc newid = match.Id; if (save == SaveTypes.CreateUpdate || save == SaveTypes.UpdateOnly) { - if (SaveEntity(container, cdEntity, match, updateinactive, updateidentical, i, unique)) + if (IsBatchable(cdEntity)) { - updated++; - references.Add(cdEntity.ToEntityReference()); + cdEntity.Id = match.Id; + var primaryIdAttribute = container.Entity(cdEntity.LogicalName).PrimaryIdAttribute; + var attrs = cdEntity.Attributes.Keys.ToList(); + if (attrs.Contains(primaryIdAttribute)) + { + attrs.Remove(primaryIdAttribute); + } + if (updateidentical || !EntityAttributesEqual(container, attrs, cdEntity, match)) + { + pendingUpdates.Add(new PendingUpdate { Entity = cdEntity, Position = i, Identifier = unique }); + if (pendingUpdates.Count >= batchsize) + { + FlushPendingUpdates(container, pendingUpdates, ref updated, ref failed, references); + } + } + else + { + skipped++; + SendLine(container, "{0:000} Skipped: {1} (Identical)", i, unique); + } } else { - skipped++; + FlushPendingCreates(container, pendingCreates, ref created, ref failed, references); + FlushPendingUpdates(container, pendingUpdates, ref updated, ref failed, references); + if (SaveEntity(container, cdEntity, match, updateinactive, updateidentical, i, unique)) + { + updated++; + references.Add(cdEntity.ToEntityReference()); + } + else + { + skipped++; + } } } else @@ -438,8 +639,7 @@ private Tuple ImportDataBloc else { failed++; - SendLine(container, $"Import object matches {matches.Count()} records in target database!"); - SendLine(container, unique); + SendLine(container, "{0:000} Match Failed: {1} matches {2} records in target database", i, unique, matches.Count()); } } if (!oldid.Equals(Guid.Empty) && !newid.Equals(Guid.Empty) && !oldid.Equals(newid) && !guidmap.ContainsKey(oldid)) @@ -448,12 +648,21 @@ private Tuple ImportDataBloc guidmap.Add(oldid, newid); } + if (deferStateAndOwner && !oldid.Equals(Guid.Empty) && !newid.Equals(Guid.Empty)) + { + UpdateDeferredActualIds(oldid, newid); + } + #endregion Entity } else if (block.Type == EntityTypes.Intersect) { #region Intersect + FlushPendingUpserts(container, pendingUpserts, ref created, ref updated, ref failed, references); + FlushPendingCreates(container, pendingCreates, ref created, ref failed, references); + FlushPendingUpdates(container, pendingUpdates, ref updated, ref failed, references); + if (cdEntity.Attributes.Count != 2) { throw new ArgumentOutOfRangeException("Attributes", cdEntity.Attributes.Count, "Invalid Attribute count for intersect object"); @@ -467,12 +676,11 @@ private Tuple ImportDataBloc var ref1 = GetAttributeEntityReference(cdEntity.Attributes.ElementAt(0)); var ref2 = GetAttributeEntityReference(cdEntity.Attributes.ElementAt(1)); - var party1 = new Entity(ref1.LogicalName, ref1.Id); //Entity.InitFromNameAndId(ref1.LogicalName, ref1.Id, crmsvc, log); - var party2 = new Entity(ref2.LogicalName, ref2.Id); //Entity.InitFromNameAndId(ref2.LogicalName, ref2.Id, crmsvc, log); + var party1 = new Entity(ref1.LogicalName, ref1.Id); + var party2 = new Entity(ref2.LogicalName, ref2.Id); try { container.Associate(party1, party2, intersect); - //party1.Associate(party2, intersect); created++; SendLine(container, "{0} Associated: {1}", i.ToString().PadLeft(3, '0'), name); } @@ -495,7 +703,7 @@ private Tuple ImportDataBloc catch (Exception ex) { failed++; - SendLine(container, $"\n*** Error record: {unique} ***\n{ex.Message}"); + SendLine(container, $"\n*** Error record: {batchFailureLabel ?? unique} ***\n{ex.Message}"); container.Log(ex); if (stoponerror) { @@ -504,6 +712,15 @@ private Tuple ImportDataBloc } i++; } + FlushPendingUpserts(container, pendingUpserts, ref created, ref updated, ref failed, references); + FlushPendingCreates(container, pendingCreates, ref created, ref failed, references); + FlushPendingUpdates(container, pendingUpdates, ref updated, ref failed, references); + + if (deferStateAndOwner) + { + FlushDeferredStateChanges(container, deferredStates); + FlushDeferredOwnerChanges(container, deferredOwners); + } SendLine(container, $"Created: {created} Updated: {updated} Skipped: {skipped} Deleted: {deleted} Failed: {failed}"); } @@ -529,6 +746,47 @@ private EntityReference GetAttributeEntityReference(KeyValuePair return null; } + /// + /// Determines whether the record holds a reference to a record that is still waiting in the + /// create batch. Such a reference cannot be remapped by yet, because + /// the referenced record has no real id until its batch is sent. + /// + private static bool ReferencesPendingCreate(Entity cdEntity, List pendingCreates) + { + if (pendingCreates.Count == 0) + { + return false; + } + foreach (var prop in cdEntity.Attributes) + { + Guid referenced; + if (prop.Value is EntityReference er) + { + referenced = er.Id; + } + else if (prop.Value is Guid guid) + { + referenced = guid; + } + else + { + continue; + } + if (referenced.Equals(Guid.Empty)) + { + continue; + } + foreach (var pending in pendingCreates) + { + if (pending.OldId.Equals(referenced)) + { + return true; + } + } + } + return false; + } + private void ReplaceGuids(IExecutionContainer container, Entity cdEntity, bool includeid) { foreach (var prop in cdEntity.Attributes) @@ -596,10 +854,11 @@ private bool SaveEntity(IExecutionContainer container, Entity cdNewEntity, Entit if (nowActive) { + var primaryIdAttribute = container.Entity(cdNewEntity.LogicalName).PrimaryIdAttribute; var updateattributes = cdNewEntity.Attributes.Keys.ToList(); - if (updateattributes.Contains(container.Entity(cdNewEntity.LogicalName).PrimaryIdAttribute)) + if (updateattributes.Contains(primaryIdAttribute)) { - updateattributes.Remove(container.Entity(cdNewEntity.LogicalName).PrimaryIdAttribute); + updateattributes.Remove(primaryIdAttribute); } if (updateIdentical || !EntityAttributesEqual(container, updateattributes, cdNewEntity, cdMatchEntity)) { @@ -609,10 +868,10 @@ private bool SaveEntity(IExecutionContainer container, Entity cdNewEntity, Entit recordSaved = true; SendLine(container, "{0:000} Updated: {1}", pos, identifier); } - catch (Exception) + catch (Exception ex) { recordSaved = false; - SendLine(container, "{0:000} Update Failed: {1} {2} {3}", pos, identifier, cdNewEntity.LogicalName); + SendLine(container, "{0:000} Update Failed: {1} {2} {3}", pos, identifier, cdNewEntity.LogicalName, ex.Message); } } else @@ -662,6 +921,1491 @@ private bool SaveEntity(IExecutionContainer container, Entity cdNewEntity, Entit return recordSaved; } + #region Batch Helpers + + private struct PendingCreate + { + public Entity Entity; + public Guid OldId; + public int Position; + public string Identifier; + } + + private struct PendingUpdate + { + public Entity Entity; + public int Position; + public string Identifier; + } + + private struct PendingUpsert + { + public Entity Entity; + public Guid OldId; + public int Position; + public string Identifier; + } + + private struct DeferredStateChange + { + public string EntityLogicalName; + public Guid OriginalId; // Id from import file (for lookup) + public Guid ActualId; // Id after create/update (for applying state) + public OptionSetValue StateCode; + public OptionSetValue StatusCode; + public int Position; + public string Identifier; + } + + private struct DeferredOwnerChange + { + public string EntityLogicalName; + public Guid OriginalId; // Id from import file (for lookup) + public Guid ActualId; // Id after create/update (for assigning owner) + public EntityReference Owner; + public int Position; + public string Identifier; + } + + /// + /// Checks if CreateMultiple message is supported for the specified entity. + /// Results are cached per entity logical name for the lifetime of the import run. + /// + /// The execution container. + /// The logical name of the entity to check. + /// True if CreateMultiple is supported; otherwise, false. + private bool IsCreateMultipleSupported(IExecutionContainer container, string entityLogicalName) + { + return IsBulkMessageSupported(container, entityLogicalName, "CreateMultiple", createMultipleSupportCache); + } + + /// + /// Checks if UpdateMultiple message is supported for the specified entity. + /// Results are cached per entity logical name for the lifetime of the import run. + /// + /// The execution container. + /// The logical name of the entity to check. + /// True if UpdateMultiple is supported; otherwise, false. + private bool IsUpdateMultipleSupported(IExecutionContainer container, string entityLogicalName) + { + return IsBulkMessageSupported(container, entityLogicalName, "UpdateMultiple", updateMultipleSupportCache); + } + + /// + /// Checks if UpsertMultiple message is supported for the specified entity. + /// Results are cached per entity logical name for the lifetime of the import run. + /// + /// The execution container. + /// The logical name of the entity to check. + /// True if UpsertMultiple is supported; otherwise, false. + private bool IsUpsertMultipleSupported(IExecutionContainer container, string entityLogicalName) + { + return IsBulkMessageSupported(container, entityLogicalName, "UpsertMultiple", upsertMultipleSupportCache); + } + + /// + /// Checks if Upsert (single) message is supported for the specified entity. + /// Results are cached per entity logical name for the lifetime of the import run. + /// + /// The execution container. + /// The logical name of the entity to check. + /// True if Upsert is supported; otherwise, false. + private bool IsUpsertSupported(IExecutionContainer container, string entityLogicalName) + { + return IsBulkMessageSupported(container, entityLogicalName, "Upsert", upsertSupportCache); + } + + /// + /// Checks if a specific SDK message is supported for an entity by querying sdkmessagefilter. + /// + /// The execution container. + /// The logical name of the entity to check. + /// The name of the SDK message (e.g., "CreateMultiple", "UpdateMultiple"). + /// The cache dictionary to use for storing results. + /// True if the message is supported; otherwise, false. + private bool IsBulkMessageSupported(IExecutionContainer container, string entityLogicalName, string messageName, Dictionary cache) + { + if (cache.TryGetValue(entityLogicalName, out var isSupported)) + { + return isSupported; + } + + try + { + var query = new QueryExpression("sdkmessagefilter") + { + ColumnSet = new ColumnSet("sdkmessagefilterid"), + TopCount = 1, + Criteria = new FilterExpression + { + FilterOperator = LogicalOperator.And, + Conditions = + { + new ConditionExpression("primaryobjecttypecode", Microsoft.Xrm.Sdk.Query.ConditionOperator.Equal, entityLogicalName) + } + }, + LinkEntities = + { + new LinkEntity + { + LinkFromEntityName = "sdkmessagefilter", + LinkToEntityName = "sdkmessage", + LinkFromAttributeName = "sdkmessageid", + LinkToAttributeName = "sdkmessageid", + LinkCriteria = new FilterExpression + { + Conditions = + { + new ConditionExpression("name", Microsoft.Xrm.Sdk.Query.ConditionOperator.Equal, messageName) + } + } + } + } + }; + + var result = container.RetrieveMultiple(query); + isSupported = result.Entities.Count > 0; + cache[entityLogicalName] = isSupported; + container.Log($"{messageName} support for {entityLogicalName}: {isSupported}"); + return isSupported; + } + catch (Exception ex) + { + container.Log($"Failed to check {messageName} support for {entityLogicalName}: {ex.Message}"); + cache[entityLogicalName] = false; + return false; + } + } + + /// + /// Marks CreateMultiple as unsupported for the specified entity (used when runtime execution fails). + /// + private void MarkCreateMultipleUnsupported(string entityLogicalName) + { + createMultipleSupportCache[entityLogicalName] = false; + } + + /// + /// Marks UpdateMultiple as unsupported for the specified entity (used when runtime execution fails). + /// + private void MarkUpdateMultipleUnsupported(string entityLogicalName) + { + updateMultipleSupportCache[entityLogicalName] = false; + } + + /// + /// Marks UpsertMultiple as unsupported for the specified entity (used when runtime execution fails). + /// + private void MarkUpsertMultipleUnsupported(string entityLogicalName) + { + upsertMultipleSupportCache[entityLogicalName] = false; + } + + /// + /// Marks Upsert (single) as unsupported for the specified entity (used when runtime execution fails). + /// + private void MarkUpsertUnsupported(string entityLogicalName) + { + upsertSupportCache[entityLogicalName] = false; + } + + /// + /// Checks if an exception indicates that the bulk message is not implemented (on-premises scenario). + /// + private static bool IsBulkMessageNotImplemented(Exception ex) + { + if (ex is FaultException fault) + { + return fault.Detail?.ErrorCode == MessageNotImplementedErrorCode; + } + if (ex is NotSupportedException) + { + return true; + } + if (ex.InnerException != null) + { + return IsBulkMessageNotImplemented(ex.InnerException); + } + return false; + } + + /// + /// The attributes DeferStateAndOwner strips off a record and applies in a second pass. + /// + private static readonly string[] stateownerattributes = { "statecode", "statuscode", "ownerid" }; + + /// + /// Determines whether a record carries anything at all besides state and owner, ignoring + /// its own primary id. Stripping the state and owner off a record that carries nothing else + /// would leave nothing to save. + /// + private static bool HasAttributesBesidesStateOwner(Entity entity) + { + var primaryid = entity.LogicalName + "id"; + return entity.Attributes.Keys.Any(a => !stateownerattributes.Contains(a) && a != primaryid); + } + + /// + /// Determines whether every record in the block carries nothing besides state and owner. + /// + private static bool IsStateOwnerOnlyBlock(EntityCollection cEntities) + { + return cEntities?.Entities.Count > 0 && !cEntities.Entities.Any(HasAttributesBesidesStateOwner); + } + + /// + /// Strips statecode, statuscode, and ownerid from an entity and defers them for later bulk application. + /// + /// The entity to strip attributes from. + /// Collection to store deferred state changes. + /// Collection to store deferred owner changes. + /// Record position for logging. + /// Record identifier for logging. + private void StripAndDeferStateOwner(Entity entity, List deferredStates, List deferredOwners, int position, string identifier) + { + if (!HasAttributesBesidesStateOwner(entity)) + { + // Nothing would be left to save. See IsStateOwnerOnlyBlock - this catches the odd + // record in a block that is otherwise worth deferring. + return; + } + + var originalId = entity.Id; + + if (entity.Contains("statecode") && entity.Contains("statuscode")) + { + deferredStates.Add(new DeferredStateChange + { + EntityLogicalName = entity.LogicalName, + OriginalId = originalId, + ActualId = Guid.Empty, // Will be updated after create/update + StateCode = entity.GetAttributeValue("statecode"), + StatusCode = entity.GetAttributeValue("statuscode"), + Position = position, + Identifier = identifier + }); + entity.Attributes.Remove("statecode"); + entity.Attributes.Remove("statuscode"); + } + + if (entity.Contains("ownerid")) + { + deferredOwners.Add(new DeferredOwnerChange + { + EntityLogicalName = entity.LogicalName, + OriginalId = originalId, + ActualId = Guid.Empty, // Will be updated after create/update + Owner = entity.GetAttributeValue("ownerid"), + Position = position, + Identifier = identifier + }); + entity.Attributes.Remove("ownerid"); + } + } + + /// + /// Updates the ActualId in deferred changes after a record is created or updated. + /// + /// The original Id from the import file. + /// The actual Id after create/update. + private void UpdateDeferredActualIds(Guid originalId, Guid actualId) + { + for (int i = 0; i < deferredStates.Count; i++) + { + if (deferredStates[i].OriginalId == originalId) + { + var item = deferredStates[i]; + item.ActualId = actualId; + deferredStates[i] = item; + } + } + + for (int i = 0; i < deferredOwners.Count; i++) + { + if (deferredOwners[i].OriginalId == originalId) + { + var item = deferredOwners[i]; + item.ActualId = actualId; + deferredOwners[i] = item; + } + } + } + + /// + /// Applies deferred state changes in bulk using UpdateMultiple when supported. + /// + /// + /// The deferred pass revisits records the main pass has already counted, so it keeps its + /// own counters and reports them on their own line. Folding them into the block totals + /// counted every deferred record twice. + /// + private void FlushDeferredStateChanges(IExecutionContainer container, List changes) + { + var notWritten = DropChangesForUnwrittenRecords(container, changes, c => c.ActualId, "state"); + if (changes.Count == 0) + { + return; + } + + container.Log($"Applying {changes.Count} deferred state changes"); + + var applied = 0; + var failed = 0; + var byEntity = changes.GroupBy(c => c.EntityLogicalName); + + foreach (var group in byEntity) + { + var entityName = group.Key; + var batch = group.ToList(); + + if (entityName == "duplicaterule" || entityName == "savedquery") + { + ApplyStatesIndividually(container, batch, ref applied, ref failed); + continue; + } + + if (IsUpdateMultipleSupported(container, entityName)) + { + if (TryApplyStatesWithUpdateMultiple(container, entityName, batch, ref applied, ref failed)) + { + continue; + } + } + + ApplyStatesIndividually(container, batch, ref applied, ref failed); + } + + SendLine(container, "Deferred state changes: {0} applied, {1} failed, {2} skipped", applied, failed, notWritten); + } + + /// + /// Removes deferred changes whose record never got an id, and says how many were dropped. + /// + /// + /// A change is still at when the main pass did not write its + /// record - no match under UpdateOnly, an ambiguous match, or nothing created. That is a + /// normal outcome the main pass has already reported, so there is nothing to apply here + /// and nothing to count as a failure. + /// + private int DropChangesForUnwrittenRecords(IExecutionContainer container, List changes, Func actualId, string kind) + { + var dropped = changes.RemoveAll(c => actualId(c) == Guid.Empty); + if (dropped > 0) + { + container.Log($"Skipping {dropped} deferred {kind} change(s) for records that were not written"); + } + + return dropped; + } + + /// + /// Attempts to apply state changes using UpdateMultiple. + /// + private bool TryApplyStatesWithUpdateMultiple(IExecutionContainer container, string entityName, List batch, ref int applied, ref int failed) + { + var targets = new EntityCollection { EntityName = entityName }; + + foreach (var change in batch) + { + var entity = new Entity(entityName, change.ActualId); + entity["statecode"] = change.StateCode; + entity["statuscode"] = change.StatusCode; + targets.Entities.Add(entity); + } + + if (targets.Entities.Count == 0) + { + return true; + } + + try + { + var request = new OrganizationRequest("UpdateMultiple") + { + Parameters = { ["Targets"] = targets } + }; + container.Service.Execute(request); + applied += targets.Entities.Count; + container.Log($"Applied {targets.Entities.Count} state changes via UpdateMultiple for {entityName}"); + return true; + } + catch (Exception ex) + { + container.Log($"UpdateMultiple for state changes failed: {ex.Message}"); + if (stoponerror) + { + throw; + } + return false; + } + } + + /// + /// Applies state changes individually using SetState. + /// + private void ApplyStatesIndividually(IExecutionContainer container, List batch, ref int applied, ref int failed) + { + for (var i = 0; i < batch.Count; i++) + { + var change = batch[i]; + try + { + var entity = new Entity(change.EntityLogicalName, change.ActualId); + + if (change.EntityLogicalName == "savedquery" && change.StateCode.Value == 1 && change.StatusCode.Value == 1) + { + container.SetState(entity, 1, 2); + } + else if (change.EntityLogicalName == "duplicaterule") + { + if (change.StatusCode.Value == 2) + { + container.PublishDuplicateRule(entity); + } + else + { + container.UnpublishDuplicateRule(entity); + } + } + else + { + container.SetState(entity, change.StateCode.Value, change.StatusCode.Value); + } + + applied++; + SendLine(container, "{0:000} SetState (deferred): {1}: {2}/{3}", change.Position, change.Identifier, change.StateCode.Value, change.StatusCode.Value); + } + catch (Exception ex) + { + failed++; + SendLine(container, "{0:000} SetState Failed (deferred): {1} {2}", change.Position, change.Identifier, ex.Message); + if (stoponerror) + { + container.Log($"StopOnError: aborting, {batch.Count - i - 1} record(s) in this batch were not executed"); + throw; + } + } + } + } + + /// + /// Applies deferred owner changes in bulk when possible. + /// + /// See for why this keeps its own counters. + private void FlushDeferredOwnerChanges(IExecutionContainer container, List changes) + { + var notWritten = DropChangesForUnwrittenRecords(container, changes, c => c.ActualId, "owner"); + if (changes.Count == 0) + { + return; + } + + container.Log($"Applying {changes.Count} deferred owner changes"); + + var applied = 0; + var failed = 0; + + for (var i = 0; i < changes.Count; i++) + { + var change = changes[i]; + try + { + var entity = new Entity(change.EntityLogicalName, change.ActualId); + container.Principal(entity).On(change.Owner).Assign(); + applied++; + SendLine(container, "{0:000} Assigned (deferred): {1} to {2} {3}", change.Position, change.Identifier, change.Owner.LogicalName, string.IsNullOrEmpty(change.Owner.Name) ? change.Owner.Id.ToString() : change.Owner.Name); + } + catch (Exception ex) + { + failed++; + SendLine(container, "{0:000} Assign Failed (deferred): {1} {2}", change.Position, change.Identifier, ex.Message); + if (stoponerror) + { + container.Log($"StopOnError: aborting, {changes.Count - i - 1} record(s) in this batch were not executed"); + throw; + } + } + } + + SendLine(container, "Deferred owner changes: {0} applied, {1} failed, {2} skipped", applied, failed, notWritten); + } + + /// + /// Records which batched record faulted and says whether the flush must abort. + /// + /// + /// Callers keep their own throw; so the original stack survives. The label is what + /// the per-record catch in reports; without it that catch + /// names whichever record happened to fill the batch. + /// + private bool StopOnBatchError(int position, string identifier) + { + if (!stoponerror) + { + return false; + } + + batchFailureLabel = string.Format("{0:000} {1}", position, identifier); + return true; + } + + + /// + /// Flushes pending create operations using CreateMultiple when supported, falling back to ExecuteMultiple or individual calls. + /// + /// The execution container. + /// The batch of pending create operations. + /// Counter for successfully created records. + /// Counter for failed records. + /// Collection to store created entity references. + private void FlushPendingCreates(IExecutionContainer container, List batch, ref int created, ref int failed, EntityReferenceCollection references) + { + if (batch.Count == 0) + { + return; + } + + if (batch.Count == 1) + { + FlushSingleCreate(container, batch[0], ref created, ref failed, references); + batch.Clear(); + return; + } + + var entityLogicalName = batch[0].Entity.LogicalName; + + if (IsCreateMultipleSupported(container, entityLogicalName)) + { + if (TryFlushCreatesWithCreateMultiple(container, batch, ref created, ref failed, references)) + { + batch.Clear(); + return; + } + } + + FlushCreatesWithExecuteMultiple(container, batch, ref created, ref failed, references); + batch.Clear(); + } + + /// + /// Creates a single record. + /// + private void FlushSingleCreate(IExecutionContainer container, PendingCreate item, ref int created, ref int failed, EntityReferenceCollection references) + { + try + { + container.Create(item.Entity); + created++; + SendLine(container, "{0:000} Created: {1}", item.Position, item.Identifier); + references.Add(item.Entity.ToEntityReference()); + RecordCreatedId(item.OldId, item.Entity.Id); + } + catch (Exception ex) + { + failed++; + SendLine(container, "{0:000} Create Failed: {1} {2}", item.Position, item.Identifier, ex.Message); + if (StopOnBatchError(item.Position, item.Identifier)) + { + throw; + } + } + } + + /// + /// Attempts to flush creates using CreateMultipleRequest (Dataverse bulk API). + /// Returns true if successful; false if the message is not supported and fallback is needed. + /// + private bool TryFlushCreatesWithCreateMultiple(IExecutionContainer container, List batch, ref int created, ref int failed, EntityReferenceCollection references) + { + var entityLogicalName = batch[0].Entity.LogicalName; + var targets = new EntityCollection { EntityName = entityLogicalName }; + foreach (var item in batch) + { + targets.Entities.Add(item.Entity); + } + + var request = new OrganizationRequest("CreateMultiple") + { + Parameters = { ["Targets"] = targets } + }; + + container.Log($"Executing CreateMultiple for {batch.Count} {entityLogicalName} records"); + + try + { + var response = container.Service.Execute(request); + var createdIds = response.Results.Contains("Ids") ? (Guid[])response.Results["Ids"] : null; + + for (var i = 0; i < batch.Count; i++) + { + var item = batch[i]; + if (createdIds != null && i < createdIds.Length) + { + item.Entity.Id = createdIds[i]; + } + created++; + SendLine(container, "{0:000} Created: {1}", item.Position, item.Identifier); + references.Add(item.Entity.ToEntityReference()); + RecordCreatedId(item.OldId, item.Entity.Id); + } + return true; + } + catch (Exception ex) + { + container.Log($"CreateMultiple failed: {ex.Message}"); + + if (IsBulkMessageNotImplemented(ex)) + { + container.Log("CreateMultiple not implemented, marking as unsupported and falling back"); + MarkCreateMultipleUnsupported(entityLogicalName); + return false; + } + + // CreateMultiple is a single transactional request, so a fault rolled the whole + // batch back and nothing was written. Re-run the rows through the per-record + // path even when StopOnError is set: that is the only way to name the record + // that actually faulted, and it restores the pre-batching behaviour of + // committing the rows ahead of it. FlushCreatesIndividually reports the + // failing row and then honours StopOnError itself. + container.Log("CreateMultiple batch failed, falling back to individual creates"); + FlushCreatesIndividually(container, batch, ref created, ref failed, references); + return true; + } + } + + /// + /// Flushes creates using ExecuteMultipleRequest (legacy batch approach). + /// + private void FlushCreatesWithExecuteMultiple(IExecutionContainer container, List batch, ref int created, ref int failed, EntityReferenceCollection references) + { + var multiRequest = new ExecuteMultipleRequest + { + Requests = new OrganizationRequestCollection(), + Settings = new ExecuteMultipleSettings + { + ContinueOnError = !stoponerror, + ReturnResponses = true + } + }; + + foreach (var item in batch) + { + multiRequest.Requests.Add(new CreateRequest { Target = item.Entity }); + } + + container.Log($"Executing ExecuteMultiple batch create of {batch.Count} records"); + + ExecuteMultipleResponse multiResponse; + try + { + multiResponse = (ExecuteMultipleResponse)container.Service.Execute(multiRequest); + } + catch (Exception ex) + { + container.Log($"ExecuteMultiple batch create failed: {ex.Message}"); + if (stoponerror) + { + throw; + } + container.Log("Falling back to sequential creates"); + FlushCreatesIndividually(container, batch, ref created, ref failed, references); + return; + } + + for (var i = 0; i < batch.Count; i++) + { + var item = batch[i]; + var responseItem = multiResponse.Responses.FirstOrDefault(r => r.RequestIndex == i); + + if (responseItem == null) + { + // ContinueOnError=false makes the platform stop at the first fault, leaving no + // response for the requests after it. Those records were never created. + failed++; + SendLine(container, "{0:000} Create Not Executed: {1}", item.Position, item.Identifier); + continue; + } + if (responseItem.Fault != null) + { + failed++; + SendLine(container, "{0:000} Create Failed: {1} {2}", item.Position, item.Identifier, responseItem.Fault.Message); + if (StopOnBatchError(item.Position, item.Identifier)) + { + container.Log($"StopOnError: aborting, {batch.Count - i - 1} record(s) in this batch were not executed"); + throw new InvalidOperationException($"Create failed: {item.Identifier} {responseItem.Fault.Message}"); + } + continue; + } + if (responseItem.Response is CreateResponse createResponse) + { + item.Entity.Id = createResponse.id; + } + created++; + SendLine(container, "{0:000} Created: {1}", item.Position, item.Identifier); + references.Add(item.Entity.ToEntityReference()); + RecordCreatedId(item.OldId, item.Entity.Id); + } + } + + /// + /// Flushes creates individually (used as fallback when batch operations fail). + /// + private void FlushCreatesIndividually(IExecutionContainer container, List batch, ref int created, ref int failed, EntityReferenceCollection references) + { + for (var i = 0; i < batch.Count; i++) + { + var item = batch[i]; + try + { + container.Create(item.Entity); + created++; + SendLine(container, "{0:000} Created: {1}", item.Position, item.Identifier); + references.Add(item.Entity.ToEntityReference()); + RecordCreatedId(item.OldId, item.Entity.Id); + } + catch (Exception itemEx) + { + failed++; + SendLine(container, "{0:000} Create Failed: {1} {2}", item.Position, item.Identifier, itemEx.Message); + if (StopOnBatchError(item.Position, item.Identifier)) + { + container.Log($"StopOnError: aborting, {batch.Count - i - 1} record(s) in this batch were not executed"); + throw; + } + } + } + } + + /// + /// Flushes pending update operations using UpdateMultiple when supported, falling back to ExecuteMultiple or individual calls. + /// + /// The execution container. + /// The batch of pending update operations. + /// Counter for successfully updated records. + /// Counter for failed records. + /// Collection to store updated entity references. + private void FlushPendingUpdates(IExecutionContainer container, List batch, ref int updated, ref int failed, EntityReferenceCollection references) + { + if (batch.Count == 0) + { + return; + } + + if (batch.Count == 1) + { + FlushSingleUpdate(container, batch[0], ref updated, ref failed, references); + batch.Clear(); + return; + } + + var entityLogicalName = batch[0].Entity.LogicalName; + + if (IsUpdateMultipleSupported(container, entityLogicalName)) + { + if (TryFlushUpdatesWithUpdateMultiple(container, batch, ref updated, ref failed, references)) + { + batch.Clear(); + return; + } + } + + FlushUpdatesWithExecuteMultiple(container, batch, ref updated, ref failed, references); + batch.Clear(); + } + + /// + /// Updates a single record. + /// + private void FlushSingleUpdate(IExecutionContainer container, PendingUpdate item, ref int updated, ref int failed, EntityReferenceCollection references) + { + try + { + container.Update(item.Entity); + updated++; + SendLine(container, "{0:000} Updated: {1}", item.Position, item.Identifier); + references.Add(item.Entity.ToEntityReference()); + } + catch (Exception ex) + { + failed++; + SendLine(container, "{0:000} Update Failed: {1} {2} {3}", item.Position, item.Identifier, item.Entity.LogicalName, ex.Message); + if (StopOnBatchError(item.Position, item.Identifier)) + { + throw; + } + } + } + + /// + /// Attempts to flush updates using UpdateMultipleRequest (Dataverse bulk API). + /// Returns true if successful; false if the message is not supported and fallback is needed. + /// + private bool TryFlushUpdatesWithUpdateMultiple(IExecutionContainer container, List batch, ref int updated, ref int failed, EntityReferenceCollection references) + { + var entityLogicalName = batch[0].Entity.LogicalName; + var targets = new EntityCollection { EntityName = entityLogicalName }; + foreach (var item in batch) + { + targets.Entities.Add(item.Entity); + } + + var request = new OrganizationRequest("UpdateMultiple") + { + Parameters = { ["Targets"] = targets } + }; + + container.Log($"Executing UpdateMultiple for {batch.Count} {entityLogicalName} records"); + + try + { + container.Service.Execute(request); + + foreach (var item in batch) + { + updated++; + SendLine(container, "{0:000} Updated: {1}", item.Position, item.Identifier); + references.Add(item.Entity.ToEntityReference()); + } + return true; + } + catch (Exception ex) + { + container.Log($"UpdateMultiple failed: {ex.Message}"); + + if (IsBulkMessageNotImplemented(ex)) + { + container.Log("UpdateMultiple not implemented, marking as unsupported and falling back"); + MarkUpdateMultipleUnsupported(entityLogicalName); + return false; + } + + // UpdateMultiple is a single transactional request, so a fault rolled the whole + // batch back and nothing was written. Re-run the rows through the per-record + // path even when StopOnError is set: that is the only way to name the record + // that actually faulted, and it restores the pre-batching behaviour of + // committing the rows ahead of it. FlushUpdatesIndividually reports the + // failing row and then honours StopOnError itself. + container.Log("UpdateMultiple batch failed, falling back to individual updates"); + FlushUpdatesIndividually(container, batch, ref updated, ref failed, references); + return true; + } + } + + /// + /// Flushes updates using ExecuteMultipleRequest (legacy batch approach). + /// + private void FlushUpdatesWithExecuteMultiple(IExecutionContainer container, List batch, ref int updated, ref int failed, EntityReferenceCollection references) + { + var multiRequest = new ExecuteMultipleRequest + { + Requests = new OrganizationRequestCollection(), + Settings = new ExecuteMultipleSettings + { + ContinueOnError = !stoponerror, + ReturnResponses = true + } + }; + + foreach (var item in batch) + { + multiRequest.Requests.Add(new UpdateRequest { Target = item.Entity }); + } + + container.Log($"Executing ExecuteMultiple batch update of {batch.Count} records"); + + ExecuteMultipleResponse multiResponse; + try + { + multiResponse = (ExecuteMultipleResponse)container.Service.Execute(multiRequest); + } + catch (Exception ex) + { + container.Log($"ExecuteMultiple batch update failed: {ex.Message}"); + if (stoponerror) + { + throw; + } + container.Log("Falling back to sequential updates"); + FlushUpdatesIndividually(container, batch, ref updated, ref failed, references); + return; + } + + for (var i = 0; i < batch.Count; i++) + { + var item = batch[i]; + var responseItem = multiResponse.Responses.FirstOrDefault(r => r.RequestIndex == i); + + if (responseItem == null) + { + // ContinueOnError=false makes the platform stop at the first fault, leaving no + // response for the requests after it. Those records were never updated. + failed++; + SendLine(container, "{0:000} Update Not Executed: {1} {2}", item.Position, item.Identifier, item.Entity.LogicalName); + continue; + } + if (responseItem.Fault != null) + { + failed++; + SendLine(container, "{0:000} Update Failed: {1} {2} {3}", item.Position, item.Identifier, item.Entity.LogicalName, responseItem.Fault.Message); + if (StopOnBatchError(item.Position, item.Identifier)) + { + container.Log($"StopOnError: aborting, {batch.Count - i - 1} record(s) in this batch were not executed"); + throw new InvalidOperationException($"Update failed: {item.Identifier} {responseItem.Fault.Message}"); + } + continue; + } + updated++; + SendLine(container, "{0:000} Updated: {1}", item.Position, item.Identifier); + references.Add(item.Entity.ToEntityReference()); + } + } + + /// + /// Flushes updates individually (used as fallback when batch operations fail). + /// + private void FlushUpdatesIndividually(IExecutionContainer container, List batch, ref int updated, ref int failed, EntityReferenceCollection references) + { + for (var i = 0; i < batch.Count; i++) + { + var item = batch[i]; + try + { + container.Update(item.Entity); + updated++; + SendLine(container, "{0:000} Updated: {1}", item.Position, item.Identifier); + references.Add(item.Entity.ToEntityReference()); + } + catch (Exception itemEx) + { + failed++; + SendLine(container, "{0:000} Update Failed: {1} {2} {3}", item.Position, item.Identifier, item.Entity.LogicalName, itemEx.Message); + if (StopOnBatchError(item.Position, item.Identifier)) + { + container.Log($"StopOnError: aborting, {batch.Count - i - 1} record(s) in this batch were not executed"); + throw; + } + } + } + } + + #region Upsert Operations + + /// + /// Flushes pending upsert operations using UpsertMultiple when supported, falling back to ExecuteMultiple with Upsert, + /// then individual Upsert, then individual Create/Update. + /// + /// The execution container. + /// The batch of pending upsert operations. + /// Counter for successfully created records. + /// Counter for successfully updated records. + /// Counter for failed records. + /// Collection to store entity references. + private void FlushPendingUpserts(IExecutionContainer container, List batch, ref int created, ref int updated, ref int failed, EntityReferenceCollection references) + { + if (batch.Count == 0) + { + return; + } + + if (batch.Count == 1) + { + FlushSingleUpsert(container, batch[0], ref created, ref updated, ref failed, references); + batch.Clear(); + return; + } + + var entityLogicalName = batch[0].Entity.LogicalName; + + // Neither Upsert nor UpsertMultiple is reported as supported for the custom entities + // tested so far, on either an on-premises 9.1 org or an online one, so both paths below + // currently fall straight through to Create/Update. They are kept because support is + // per-entity: several out-of-the-box tables already carry Upsert, and Microsoft enables + // the bulk messages on more tables over time. Detection is per entity and cached, so an + // org that gains support starts using it with no change here. + if (IsUpsertMultipleSupported(container, entityLogicalName)) + { + if (TryFlushUpsertsWithUpsertMultiple(container, batch, ref created, ref updated, ref failed, references)) + { + batch.Clear(); + return; + } + } + + if (IsUpsertSupported(container, entityLogicalName)) + { + if (TryFlushUpsertsWithExecuteMultiple(container, batch, ref created, ref updated, ref failed, references)) + { + batch.Clear(); + return; + } + } + + // Final fallback: individual Create/Update operations + FlushUpsertsAsCreateUpdate(container, batch, ref created, ref updated, ref failed, references); + batch.Clear(); + } + + /// + /// Upserts a single record. + /// + private void FlushSingleUpsert(IExecutionContainer container, PendingUpsert item, ref int created, ref int updated, ref int failed, EntityReferenceCollection references) + { + var entityLogicalName = item.Entity.LogicalName; + + if (IsUpsertSupported(container, entityLogicalName)) + { + try + { + var request = new UpsertRequest { Target = item.Entity }; + var response = (UpsertResponse)container.Service.Execute(request); + + if (response.RecordCreated) + { + if (response.Target != null) + { + item.Entity.Id = response.Target.Id; + } + created++; + SendLine(container, "{0:000} Created (upsert): {1}", item.Position, item.Identifier); + } + else + { + updated++; + SendLine(container, "{0:000} Updated (upsert): {1}", item.Position, item.Identifier); + } + references.Add(item.Entity.ToEntityReference()); + MapGuid(item.OldId, item.Entity.Id); + return; + } + catch (Exception ex) + { + if (IsBulkMessageNotImplemented(ex)) + { + container.Log($"Upsert not implemented for {entityLogicalName}, falling back to Create/Update"); + MarkUpsertUnsupported(entityLogicalName); + } + else + { + failed++; + SendLine(container, "{0:000} Upsert Failed: {1} {2}", item.Position, item.Identifier, ex.Message); + if (StopOnBatchError(item.Position, item.Identifier)) + { + throw; + } + return; + } + } + } + + // Fallback to Create (since we don't have a match for single upsert fallback) + try + { + container.Create(item.Entity); + created++; + SendLine(container, "{0:000} Created: {1}", item.Position, item.Identifier); + references.Add(item.Entity.ToEntityReference()); + MapGuid(item.OldId, item.Entity.Id); + } + catch (Exception ex) + { + failed++; + SendLine(container, "{0:000} Create Failed: {1} {2}", item.Position, item.Identifier, ex.Message); + if (StopOnBatchError(item.Position, item.Identifier)) + { + throw; + } + } + } + + /// + /// Attempts to flush upserts using UpsertMultipleRequest (Dataverse bulk API). + /// Returns true if successful; false if the message is not supported and fallback is needed. + /// + private bool TryFlushUpsertsWithUpsertMultiple(IExecutionContainer container, List batch, ref int created, ref int updated, ref int failed, EntityReferenceCollection references) + { + var entityLogicalName = batch[0].Entity.LogicalName; + var targets = new EntityCollection { EntityName = entityLogicalName }; + foreach (var item in batch) + { + targets.Entities.Add(item.Entity); + } + + var request = new OrganizationRequest("UpsertMultiple") + { + Parameters = { ["Targets"] = targets } + }; + + container.Log($"Executing UpsertMultiple for {batch.Count} {entityLogicalName} records"); + + try + { + var response = container.Service.Execute(request); + var results = response.Results.Contains("Results") ? (UpsertResponse[])response.Results["Results"] : null; + + for (var i = 0; i < batch.Count; i++) + { + var item = batch[i]; + var upsertResult = results != null && i < results.Length ? results[i] : null; + + if (upsertResult != null) + { + if (upsertResult.RecordCreated) + { + if (upsertResult.Target != null) + { + item.Entity.Id = upsertResult.Target.Id; + } + created++; + SendLine(container, "{0:000} Created (upsert): {1}", item.Position, item.Identifier); + } + else + { + updated++; + SendLine(container, "{0:000} Updated (upsert): {1}", item.Position, item.Identifier); + } + } + else + { + // If no result available, count as updated (default upsert behavior) + updated++; + SendLine(container, "{0:000} Upserted: {1}", item.Position, item.Identifier); + } + references.Add(item.Entity.ToEntityReference()); + MapGuid(item.OldId, item.Entity.Id); + } + return true; + } + catch (Exception ex) + { + container.Log($"UpsertMultiple failed: {ex.Message}"); + + if (IsBulkMessageNotImplemented(ex)) + { + container.Log("UpsertMultiple not implemented, marking as unsupported and falling back"); + MarkUpsertMultipleUnsupported(entityLogicalName); + return false; + } + + // UpsertMultiple is a single transactional request, so a fault rolled the whole + // batch back and nothing was written. Re-run the rows through the per-record + // path even when StopOnError is set: that is the only way to name the record + // that actually faulted, and it restores the pre-batching behaviour of + // committing the rows ahead of it. TryFlushUpsertsWithExecuteMultiple reports the + // failing row and then honours StopOnError itself. + container.Log("UpsertMultiple batch failed, falling back to ExecuteMultiple with Upsert"); + // Try ExecuteMultiple with individual Upsert requests + return TryFlushUpsertsWithExecuteMultiple(container, batch, ref created, ref updated, ref failed, references); + } + } + + /// + /// Flushes upserts using ExecuteMultipleRequest with individual UpsertRequest items. + /// Returns true if successful; false if Upsert is not supported and fallback is needed. + /// + private bool TryFlushUpsertsWithExecuteMultiple(IExecutionContainer container, List batch, ref int created, ref int updated, ref int failed, EntityReferenceCollection references) + { + var entityLogicalName = batch[0].Entity.LogicalName; + var multiRequest = new ExecuteMultipleRequest + { + Requests = new OrganizationRequestCollection(), + Settings = new ExecuteMultipleSettings + { + ContinueOnError = !stoponerror, + ReturnResponses = true + } + }; + + foreach (var item in batch) + { + multiRequest.Requests.Add(new UpsertRequest { Target = item.Entity }); + } + + container.Log($"Executing ExecuteMultiple with UpsertRequest for {batch.Count} {entityLogicalName} records"); + + ExecuteMultipleResponse multiResponse; + try + { + multiResponse = (ExecuteMultipleResponse)container.Service.Execute(multiRequest); + } + catch (Exception ex) + { + container.Log($"ExecuteMultiple with Upsert failed: {ex.Message}"); + + if (IsBulkMessageNotImplemented(ex)) + { + container.Log("Upsert not implemented, marking as unsupported and falling back"); + MarkUpsertUnsupported(entityLogicalName); + return false; + } + + // The request itself failed, so no item was applied. Upsert is idempotent, so + // re-running the batch as Create/Update is safe and is the only way to name the + // failing record. FlushUpsertsAsCreateUpdate honours StopOnError itself. + container.Log("Falling back to individual Create/Update operations"); + FlushUpsertsAsCreateUpdate(container, batch, ref created, ref updated, ref failed, references); + return true; + } + + for (var i = 0; i < batch.Count; i++) + { + var item = batch[i]; + var responseItem = multiResponse.Responses.FirstOrDefault(r => r.RequestIndex == i); + + if (responseItem == null) + { + // ContinueOnError=false makes the platform stop at the first fault, leaving no + // response for the requests after it. Those records were never upserted. + failed++; + SendLine(container, "{0:000} Upsert Not Executed: {1}", item.Position, item.Identifier); + continue; + } + if (responseItem.Fault != null) + { + // Check if fault indicates Upsert not implemented + if (responseItem.Fault.ErrorCode == MessageNotImplementedErrorCode) + { + container.Log("Upsert not implemented, marking as unsupported and falling back to Create/Update"); + MarkUpsertUnsupported(entityLogicalName); + return false; + } + failed++; + SendLine(container, "{0:000} Upsert Failed: {1} {2}", item.Position, item.Identifier, responseItem.Fault.Message); + if (StopOnBatchError(item.Position, item.Identifier)) + { + container.Log($"StopOnError: aborting, {batch.Count - i - 1} record(s) in this batch were not executed"); + throw new InvalidOperationException($"Upsert failed: {item.Identifier} {responseItem.Fault.Message}"); + } + continue; + } + if (!(responseItem.Response is UpsertResponse upsertResponse)) + { + failed++; + SendLine(container, "{0:000} Upsert Failed: {1} unexpected response {2}", item.Position, item.Identifier, responseItem.Response?.GetType().Name ?? "(none)"); + continue; + } + if (upsertResponse.RecordCreated) + { + if (upsertResponse.Target != null) + { + item.Entity.Id = upsertResponse.Target.Id; + } + created++; + SendLine(container, "{0:000} Created (upsert): {1}", item.Position, item.Identifier); + } + else + { + updated++; + SendLine(container, "{0:000} Updated (upsert): {1}", item.Position, item.Identifier); + } + references.Add(item.Entity.ToEntityReference()); + MapGuid(item.OldId, item.Entity.Id); + } + + return true; + } + + /// + /// Flushes upserts using individual Create/Update operations (final fallback for CRM 8.x and older). + /// Since we don't have match results, we attempt Create first and fall back to Update on duplicate key error. + /// + private void FlushUpsertsAsCreateUpdate(IExecutionContainer container, List batch, ref int created, ref int updated, ref int failed, EntityReferenceCollection references) + { + container.Log($"Falling back to Create/Update for {batch.Count} records (Upsert not available)"); + + for (var i = 0; i < batch.Count; i++) + { + var item = batch[i]; + try + { + // Attempt Create first + container.Create(item.Entity); + created++; + SendLine(container, "{0:000} Created: {1}", item.Position, item.Identifier); + references.Add(item.Entity.ToEntityReference()); + MapGuid(item.OldId, item.Entity.Id); + } + catch (FaultException createEx) + { + // Check for duplicate key error (record exists) + if (createEx.Detail?.ErrorCode == -2147220937 || // DuplicateRecordEntityKey + createEx.Detail?.ErrorCode == -2147220685 || // DuplicateRecord + createEx.Message.Contains("duplicate") || + createEx.Message.Contains("already exists")) + { + // Record exists, try Update instead + try + { + container.Update(item.Entity); + updated++; + SendLine(container, "{0:000} Updated: {1}", item.Position, item.Identifier); + references.Add(item.Entity.ToEntityReference()); + MapGuid(item.OldId, item.Entity.Id); + } + catch (Exception updateEx) + { + failed++; + SendLine(container, "{0:000} Update Failed (fallback): {1} {2}", item.Position, item.Identifier, updateEx.Message); + if (StopOnBatchError(item.Position, item.Identifier)) + { + container.Log($"StopOnError: aborting, {batch.Count - i - 1} record(s) in this batch were not executed"); + throw; + } + } + } + else + { + failed++; + SendLine(container, "{0:000} Create Failed: {1} {2}", item.Position, item.Identifier, createEx.Message); + if (StopOnBatchError(item.Position, item.Identifier)) + { + container.Log($"StopOnError: aborting, {batch.Count - i - 1} record(s) in this batch were not executed"); + throw; + } + } + } + catch (Exception ex) + { + failed++; + SendLine(container, "{0:000} Create Failed: {1} {2}", item.Position, item.Identifier, ex.Message); + if (StopOnBatchError(item.Position, item.Identifier)) + { + container.Log($"StopOnError: aborting, {batch.Count - i - 1} record(s) in this batch were not executed"); + throw; + } + } + } + } + + #endregion Upsert Operations + + private void FlushPendingDeletes(IExecutionContainer container, List batch, ref int deleted, ref int failed) + { + if (batch.Count == 0) + { + return; + } + if (batch.Count == 1) + { + container.Delete(batch[0]); + deleted++; + batch.Clear(); + return; + } + var multiRequest = new ExecuteMultipleRequest + { + Requests = new OrganizationRequestCollection(), + Settings = new ExecuteMultipleSettings + { + ContinueOnError = !stoponerror, + ReturnResponses = true + } + }; + foreach (var entity in batch) + { + multiRequest.Requests.Add(new DeleteRequest { Target = entity.ToEntityReference() }); + } + container.Log($"Executing batch delete of {batch.Count} records"); + ExecuteMultipleResponse multiResponse; + try + { + multiResponse = (ExecuteMultipleResponse)container.Service.Execute(multiRequest); + } + catch (Exception ex) + { + container.Log($"Batch delete failed: {ex.Message}"); + container.Log("Falling back to sequential deletes"); + foreach (var entity in batch) + { + try + { + container.Delete(entity); + deleted++; + } + catch (FaultException fex) + { + if (fex.Message.ToUpperInvariant().Contains("DOES NOT EXIST")) + { + SendLine(container, " ...already deleted"); + } + else + { + throw; + } + } + } + batch.Clear(); + return; + } + + for (var i = 0; i < batch.Count; i++) + { + var responseItem = multiResponse.Responses.FirstOrDefault(r => r.RequestIndex == i); + if (responseItem == null) + { + // ContinueOnError=false makes the platform stop at the first fault, leaving no + // response for the requests after it. Those records were never deleted. + failed++; + SendLine(container, "Delete Not Executed: {0}", batch[i].LogicalName); + continue; + } + if (responseItem.Fault != null) + { + if (responseItem.Fault.Message.ToUpperInvariant().Contains("DOES NOT EXIST")) + { + SendLine(container, " ...already deleted"); + } + else + { + failed++; + SendLine(container, "Delete Failed: {0} {1}", batch[i].LogicalName, responseItem.Fault.Message); + if (stoponerror) + { + container.Log($"StopOnError: aborting, {batch.Count - i - 1} record(s) in this batch were not executed"); + throw new InvalidOperationException($"Delete failed: {batch[i].LogicalName} {responseItem.Fault.Message}"); + } + } + continue; + } + deleted++; + } + batch.Clear(); + } + + private void MapGuid(Guid oldId, Guid newId) + { + if (!oldId.Equals(Guid.Empty) && !newId.Equals(Guid.Empty) && !oldId.Equals(newId) && !guidmap.ContainsKey(oldId)) + { + guidmap.Add(oldId, newId); + } + } + + /// + /// Records the actual id of a created record: maps it for later lookup remapping, and fills in + /// the id of any deferred state or owner change waiting for that record. + /// The import loop only does this itself for records it created inline; a record created from a + /// batch has no id at that point, so the flush methods must do it here instead. + /// This is deliberately not folded into : the guid map skips ids that are + /// unchanged or already mapped, but a deferred change still needs its id in both those cases. + /// + private void RecordCreatedId(Guid oldId, Guid newId) + { + MapGuid(oldId, newId); + if (!oldId.Equals(Guid.Empty) && !newId.Equals(Guid.Empty)) + { + UpdateDeferredActualIds(oldId, newId); + } + } + + /// + /// Determines if a record can be saved with a simple Create or Update (no state changes, no owner reassignment). + /// + private static bool IsBatchable(Entity entity) + { + return !entity.Contains("statecode") && !entity.Contains("statuscode") && !entity.Contains("ownerid"); + } + + #endregion Batch Helpers + #endregion Private Methods } diff --git a/shared/Xrm.Shuffle.Core/ShuffleSolutionImport.cs b/shared/Xrm.Shuffle.Core/ShuffleSolutionImport.cs index 1500505..b339410 100644 --- a/shared/Xrm.Shuffle.Core/ShuffleSolutionImport.cs +++ b/shared/Xrm.Shuffle.Core/ShuffleSolutionImport.cs @@ -163,12 +163,13 @@ private bool DoImportSolutionAsync(ImportSolutionRequest impSolReq, ref Exceptio } if (cdAsyncOperation != null) { - container.Attribute(SystemJob.StatusReason).On(cdAsyncOperation).ToString(); - statustext = container.Attribute(SystemJob.StatusReason).On(cdAsyncOperation).ToString(); var newStatus = cdAsyncOperation.GetAttribute(SystemJob.StatusReason, new OptionSetValue()).Value; if (newStatus != importStatus) { importStatus = newStatus; + statustext = cdAsyncOperation.FormattedValues.Contains(SystemJob.StatusReason) + ? cdAsyncOperation.FormattedValues[SystemJob.StatusReason] + : importStatus.ToString(); if (end.Equals(DateTime.MaxValue) && importStatus != (int)SystemJob.StatusReason_OptionSet.Waiting) { end = timeout > 0 ? DateTime.Now.AddMinutes(timeout) : DateTime.Now.AddMinutes(2); @@ -208,9 +209,13 @@ private bool DoImportSolutionAsync(ImportSolutionRequest impSolReq, ref Exceptio SendLine(container, "See log file for technical details."); } } - container.Attribute(SystemJob.Status).On(cdAsyncOperation).ToString(); - - ex = new Exception($"Solution Import Failed: {container.Attribute(SystemJob.Status).On(cdAsyncOperation).ToString()} - {container.Attribute(SystemJob.StatusReason).On(cdAsyncOperation).ToString()}"); + var statusLabel = cdAsyncOperation.FormattedValues.Contains(SystemJob.Status) + ? cdAsyncOperation.FormattedValues[SystemJob.Status] + : cdAsyncOperation.GetAttribute(SystemJob.Status, new OptionSetValue()).Value.ToString(); + var reasonLabel = cdAsyncOperation.FormattedValues.Contains(SystemJob.StatusReason) + ? cdAsyncOperation.FormattedValues[SystemJob.StatusReason] + : importStatus.ToString(); + ex = new Exception($"Solution Import Failed: {statusLabel} - {reasonLabel}"); break; } @@ -481,7 +486,6 @@ private void ValidatePreReqs(IExecutionContainer container, SolutionBlockImport var name = prereq.Name; var comparer = prereq.Comparer; var version = new Version(); - container.Log("Prereq: {0} {1} {2}", name, comparer, version); if (comparer == SolutionVersionComparers.eqthis || comparer == SolutionVersionComparers.gethis) { @@ -493,6 +497,10 @@ private void ValidatePreReqs(IExecutionContainer container, SolutionBlockImport version = new Version(prereq.Version.Replace('*', '0')); } + // Logged after resolution - comparer and version are both rewritten above, + // so logging first reported "ge 0.0" for every prerequisite. + container.Log("Prereq: {0} {1} {2}", name, comparer, version); + foreach (var cdSolution in cSolutions.Entities) { if (cdSolution.GetAttribute("uniquename", "") == name) diff --git a/shared/Xrm.Shuffle.Core/Shuffler.cs b/shared/Xrm.Shuffle.Core/Shuffler.cs index b7dd4f3..373d752 100644 --- a/shared/Xrm.Shuffle.Core/Shuffler.cs +++ b/shared/Xrm.Shuffle.Core/Shuffler.cs @@ -556,7 +556,7 @@ private void SendText(IExecutionContainer container, string msg, int totalBlocks msg = string.Format(msg, args); if (msg.Length > 1) { - container.Log(msg, args); + container.Log(msg); } } OnRaiseShuffleEvent(new ShuffleEventArgs(msg, totalBlocks, currentBlock, blockRecords, currentRecord, replacelast)); diff --git a/tests/Xrm.Shuffle.Core.Tests/Helpers/DataXml.cs b/tests/Xrm.Shuffle.Core.Tests/Helpers/DataXml.cs new file mode 100644 index 0000000..3443b36 --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Helpers/DataXml.cs @@ -0,0 +1,196 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; +using System.Xml; + +namespace Cinteros.Crm.Utils.Shuffle.Tests.Helpers +{ + /// + /// Builds the <ShuffleData> documents that Shuffler.Deserialize reads. + /// + /// + /// Only the Simple serialization type is produced. That matters: deserializing a Simple + /// attribute is a pure switch over the type string + /// (Xrm.Utils.Core.Common/Extensions/EntityExtensions.cs, SetAttribute), so test + /// data needs no organization service and no metadata call. Explicit and + /// SimpleWithValue would both reach for metadata, which is why fixtures stay on Simple. + /// + public class DataXml + { + private readonly List blocks = new List(); + + /// Starts a document with one block already open. + public static BlockBuilder Block(string name) + { + return new DataXml().WithBlock(name); + } + + /// Opens another block in this document. + public BlockBuilder WithBlock(string name) + { + var block = new BlockBuilder(this, name); + blocks.Add(block); + return block; + } + + /// Renders the document. + public XmlDocument Build() + { + var xml = new StringBuilder(); + xml.Append(""); + foreach (var block in blocks) + { + block.AppendTo(xml); + } + xml.Append(""); + var document = new XmlDocument(); + document.LoadXml(xml.ToString()); + return document; + } + + /// One <Block> and the records in it. + public class BlockBuilder + { + private readonly DataXml owner; + private readonly string name; + private readonly List records = new List(); + + internal BlockBuilder(DataXml owner, string name) + { + this.owner = owner; + this.name = name; + } + + /// Adds a record. A null id emits no id attribute at all. + public RecordBuilder Record(string entityLogicalName, Guid? id = null) + { + var record = new RecordBuilder(this, entityLogicalName, id); + records.Add(record); + return record; + } + + /// Opens a sibling block in the same document. + public BlockBuilder AndBlock(string blockName) + { + return owner.WithBlock(blockName); + } + + /// Renders the whole document, not just this block. + public XmlDocument Build() + { + return owner.Build(); + } + + internal void AppendTo(StringBuilder xml) + { + xml.Append(""); + foreach (var record in records) + { + record.AppendTo(xml); + } + xml.Append(""); + } + } + + /// One <Entity> and its attributes. + public class RecordBuilder + { + private readonly BlockBuilder owner; + private readonly string entityLogicalName; + private readonly Guid? id; + private readonly List attributes = new List(); + + internal RecordBuilder(BlockBuilder owner, string entityLogicalName, Guid? id) + { + this.owner = owner; + this.entityLogicalName = entityLogicalName; + this.id = id; + } + + /// Adds a string attribute. + public RecordBuilder With(string attribute, string value) + { + return With(attribute, "String", value); + } + + /// Adds an attribute of an explicit type, as the serializer would write it. + public RecordBuilder With(string attribute, string type, string value) + { + attributes.Add(string.Format( + CultureInfo.InvariantCulture, + "{2}", + Escape(attribute), Escape(type), Escape(value))); + return this; + } + + /// Adds an integer attribute. + public RecordBuilder WithInt(string attribute, int value) + { + return With(attribute, "Int32", value.ToString(CultureInfo.InvariantCulture)); + } + + /// Adds an optionset attribute. + public RecordBuilder WithOptionSet(string attribute, int value) + { + return With(attribute, "OptionSetValue", value.ToString(CultureInfo.InvariantCulture)); + } + + /// Adds a lookup. The serializer writes the target in an entity attribute. + public RecordBuilder WithReference(string attribute, string targetLogicalName, Guid targetId) + { + attributes.Add(string.Format( + CultureInfo.InvariantCulture, + "{2}", + Escape(attribute), Escape(targetLogicalName), targetId)); + return this; + } + + /// Adds another record to the same block. + public RecordBuilder AndRecord(string logicalName, Guid? recordId = null) + { + return owner.Record(logicalName, recordId); + } + + /// Opens a sibling block in the same document. + public BlockBuilder AndBlock(string blockName) + { + return owner.AndBlock(blockName); + } + + /// Renders the whole document. + public XmlDocument Build() + { + return owner.Build(); + } + + internal void AppendTo(StringBuilder xml) + { + xml.Append(""); + foreach (var attribute in attributes) + { + xml.Append(attribute); + } + xml.Append(""); + } + } + + private static string Escape(string value) + { + if (string.IsNullOrEmpty(value)) + { + return string.Empty; + } + return value + .Replace("&", "&") + .Replace("<", "<") + .Replace(">", ">") + .Replace("\"", """); + } + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Helpers/DefinitionXml.cs b/tests/Xrm.Shuffle.Core.Tests/Helpers/DefinitionXml.cs new file mode 100644 index 0000000..b9a7ebc --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Helpers/DefinitionXml.cs @@ -0,0 +1,216 @@ +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text; +using System.Xml; +using System.Xml.Serialization; +using Cinteros.Crm.Utils.Shuffle.Types; + +namespace Cinteros.Crm.Utils.Shuffle.Tests.Helpers +{ + /// + /// Builds <ShuffleDefinition> documents, and the deserialized + /// ShuffleDefinition objects the import code actually reads. + /// + /// + /// ShuffleDefinition.xsd declares no targetNamespace, so the literals here carry + /// no namespace declaration and none should be added — the generated serializer would then + /// fail to match the root element. + /// + public class DefinitionXml + { + private readonly List blocks = new List(); + private bool? stopOnError; + private int? timeout; + + /// Starts a definition with one data block already open. + public static DataBlockBuilder DataBlock(string name, string entityLogicalName) + { + return new DefinitionXml().WithDataBlock(name, entityLogicalName); + } + + /// Opens another data block in this definition. + public DataBlockBuilder WithDataBlock(string name, string entityLogicalName) + { + var block = new DataBlockBuilder(this, name, entityLogicalName); + blocks.Add(block); + return block; + } + + /// Sets the definition-level StopOnError attribute. + public DefinitionXml StopOnError(bool value) + { + stopOnError = value; + return this; + } + + /// Sets the definition-level Timeout attribute. + public DefinitionXml Timeout(int seconds) + { + timeout = seconds; + return this; + } + + /// Renders the definition as XML. + public XmlDocument Build() + { + var xml = new StringBuilder(); + xml.Append(""); + foreach (var block in blocks) + { + block.AppendTo(xml); + } + xml.Append(""); + var document = new XmlDocument(); + document.LoadXml(xml.ToString()); + return document; + } + + /// Renders the definition and deserializes it the way Shuffler does. + public ShuffleDefinition Deserialize() + { + var serializer = new XmlSerializer(typeof(ShuffleDefinition)); + using (var reader = new StringReader(Build().OuterXml)) + { + return (ShuffleDefinition)serializer.Deserialize(reader); + } + } + + /// Renders the definition and returns its single data block. + public global::Cinteros.Crm.Utils.Shuffle.Types.DataBlock DeserializeBlock() + { + return (global::Cinteros.Crm.Utils.Shuffle.Types.DataBlock)Deserialize().Blocks.Items[0]; + } + + /// One <DataBlock> and its <Import> settings. + public class DataBlockBuilder + { + private readonly DefinitionXml owner; + private readonly string name; + private readonly string entityLogicalName; + private readonly List importAttributes = new List(); + private readonly List matchAttributes = new List(); + private bool import; + private bool match; + private bool preRetrieveAll; + + internal DataBlockBuilder(DefinitionXml owner, string name, string entityLogicalName) + { + this.owner = owner; + this.name = name; + this.entityLogicalName = entityLogicalName; + } + + /// Adds an <Import> element with no attributes set. + public DataBlockBuilder Import() + { + import = true; + return this; + } + + /// Sets any Import attribute by name, e.g. Save or Delete. + public DataBlockBuilder ImportAttribute(string attribute, string value) + { + import = true; + importAttributes.Add(string.Format( + CultureInfo.InvariantCulture, " {0}=\"{1}\"", attribute, value)); + return this; + } + + /// Sets BatchSize. Omitting it is what leaves batching off. + public DataBlockBuilder BatchSize(int size) + { + return ImportAttribute("BatchSize", size.ToString(CultureInfo.InvariantCulture)); + } + + /// Sets DeferStateAndOwner. + public DataBlockBuilder DeferStateAndOwner(bool value = true) + { + return ImportAttribute("DeferStateAndOwner", value ? "true" : "false"); + } + + /// Sets CreateWithId. + public DataBlockBuilder CreateWithId(bool value = true) + { + return ImportAttribute("CreateWithId", value ? "true" : "false"); + } + + /// + /// Adds a match attribute. drives PreRetrieveAll, which is + /// the switch that lets a block reach any batched path at all. + /// + public DataBlockBuilder MatchOn(string attribute, bool retrieveAll = true) + { + import = true; + match = true; + preRetrieveAll = preRetrieveAll || retrieveAll; + matchAttributes.Add(string.Format( + CultureInfo.InvariantCulture, "", attribute)); + return this; + } + + /// Opens a sibling data block. + public DataBlockBuilder AndDataBlock(string blockName, string blockEntity) + { + return owner.WithDataBlock(blockName, blockEntity); + } + + /// Renders the whole definition as XML. + public XmlDocument Build() + { + return owner.Build(); + } + + /// Renders and deserializes the whole definition. + public ShuffleDefinition Deserialize() + { + return owner.Deserialize(); + } + + /// Renders and returns the first data block. + public global::Cinteros.Crm.Utils.Shuffle.Types.DataBlock DeserializeBlock() + { + return owner.DeserializeBlock(); + } + + internal void AppendTo(StringBuilder xml) + { + xml.Append(""); + if (import) + { + xml.Append(""); + xml.Append(""); + foreach (var attribute in matchAttributes) + { + xml.Append(attribute); + } + xml.Append(""); + } + else + { + xml.Append(" />"); + } + } + xml.Append(""); + } + } + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Helpers/ExecuteMultipleResponseBuilder.cs b/tests/Xrm.Shuffle.Core.Tests/Helpers/ExecuteMultipleResponseBuilder.cs new file mode 100644 index 0000000..f0fa929 --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Helpers/ExecuteMultipleResponseBuilder.cs @@ -0,0 +1,153 @@ +namespace Cinteros.Crm.Utils.Shuffle.Tests.Helpers +{ + using System; + using System.Collections.Generic; + using System.ServiceModel; + using Microsoft.Xrm.Sdk; + using Microsoft.Xrm.Sdk.Messages; + + /// + /// Builds an item by item, so a fixture can + /// describe exactly which requests in a batch succeeded, which faulted, and which + /// the platform did not answer at all. + /// + /// + /// + /// ExecuteMultipleResponse has no public API for adding results - its Responses and + /// IsFaulted properties are read-only projections over the Results bag - so this writes + /// the two well-known keys directly. ExecuteMultipleResponseItem itself has a public + /// parameterless constructor and public setters on all four properties, so no reflection + /// is needed. + /// + /// + /// is the reason this class exists. The platform is documented to + /// return one response item per request, but the product code defends against a short + /// collection anyway; FakeXrmEasy cannot express that, so the scripted service must. + /// + /// + public class ExecuteMultipleResponseBuilder + { + private readonly List items = new List(); + private int next; + + /// A response item carrying for the next request. + public ExecuteMultipleResponseBuilder Success(OrganizationResponse response) + { + items.Add(new ExecuteMultipleResponseItem + { + RequestIndex = next++, + Response = response + }); + return this; + } + + /// A carrying . + public ExecuteMultipleResponseBuilder CreatedAt(Guid id) + { + var response = new CreateResponse(); + response.Results["id"] = id; + return Success(response); + } + + /// An empty success - what Update, Upsert and Delete return. + public ExecuteMultipleResponseBuilder Succeeded() + { + return Success(new OrganizationResponse()); + } + + /// A fault for the next request, with . + public ExecuteMultipleResponseBuilder Fault(string message) + { + return Fault(message, -2147220989); + } + + /// A fault for the next request, with an explicit error code. + public ExecuteMultipleResponseBuilder Fault(string message, int errorCode) + { + items.Add(new ExecuteMultipleResponseItem + { + RequestIndex = next++, + Fault = new OrganizationServiceFault { Message = message, ErrorCode = errorCode } + }); + return this; + } + + /// + /// Advances the request index without adding an item, leaving a hole the product + /// code has to survive. + /// + public ExecuteMultipleResponseBuilder Omit() + { + next++; + return this; + } + + /// + /// Adds a fault for an explicit request index, out of order - the flush loops look + /// items up by RequestIndex rather than by position, and that has to keep holding. + /// + public ExecuteMultipleResponseBuilder FaultAt(int requestIndex, string message) + { + items.Add(new ExecuteMultipleResponseItem + { + RequestIndex = requestIndex, + Fault = new OrganizationServiceFault { Message = message, ErrorCode = -2147220989 } + }); + if (requestIndex >= next) + { + next = requestIndex + 1; + } + return this; + } + + /// Adds a success for an explicit request index, out of order. + public ExecuteMultipleResponseBuilder SuccessAt(int requestIndex, OrganizationResponse response) + { + items.Add(new ExecuteMultipleResponseItem + { + RequestIndex = requestIndex, + Response = response + }); + if (requestIndex >= next) + { + next = requestIndex + 1; + } + return this; + } + + /// The assembled response. + public ExecuteMultipleResponse Build() + { + var collection = new ExecuteMultipleResponseItemCollection(); + collection.AddRange(items); + + var response = new ExecuteMultipleResponse(); + response.Results["Responses"] = collection; + response.Results["IsFaulted"] = items.Exists(i => i.Fault != null); + return response; + } + + /// A fault the platform raises when a message is not implemented at all. + /// + /// 0x80040265 is what the product watches for to decide a bulk message is unavailable + /// and fall back - see MessageNotImplementedErrorCode in ShuffleDataImport. + /// + public static FaultException MessageNotImplemented(string messageName) + { + return new FaultException( + new OrganizationServiceFault + { + Message = $"The request message '{messageName}' is not implemented.", + ErrorCode = unchecked((int)0x80040265) + }); + } + + /// An ordinary platform fault, as a thrown exception rather than a batch item. + public static FaultException Faulted(string message) + { + return new FaultException( + new OrganizationServiceFault { Message = message, ErrorCode = -2147220989 }, + message); + } + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Helpers/FakeOrgTestBase.cs b/tests/Xrm.Shuffle.Core.Tests/Helpers/FakeOrgTestBase.cs new file mode 100644 index 0000000..4034857 --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Helpers/FakeOrgTestBase.cs @@ -0,0 +1,132 @@ +namespace Cinteros.Crm.Utils.Shuffle.Tests.Helpers +{ + using System; + using System.Linq; + using Microsoft.Xrm.Sdk; + using NUnit.Framework; + + /// + /// Base for the fixtures that run against a fake organization rather than a scripted one. + /// + /// + /// + /// Layer 1 scripts every answer, which is what the response-pairing tests need and what + /// makes them unreadable as descriptions of ordinary behaviour. These fixtures go the other + /// way: a plausible org, seeded with rows, and a whole data block run through it. What they + /// test is the decisions taken before any flush happens - whether a block is batchable, + /// which records it matched, whether the upsert gate opened, which bulk message the + /// capability probe chose. + /// + /// + /// Each test builds its own org, because the shape of the org is the arrangement: + /// OnPrem() and Online() differ only in what the capability probe answers, + /// and that difference is what most of these tests are about. Seeding has to finish before + /// the container is asked for - see . + /// + /// + public abstract class FakeOrgTestBase + { + /// The org the current test built, once it has built one. + protected ShuffleTestContext Org { get; private set; } + + /// Both output channels, once a shuffler has been built. + protected ShuffleEventRecorder Recorder { get; private set; } + + /// The recording service, for asserting which requests were sent. + protected RecordingOrganizationService Service + { + get { return Org.Service; } + } + + /// Fails the fixture early if the cache reset has stopped finding its fields. + [OneTimeSetUp] + public void AssertCachesAreReachable() + { + SharedStaticCaches.AssertResolved(); + } + + /// + /// Empty caches and no org before every test. The attribute-name dictionaries in + /// Xrm.Utils.Core.Common are process-wide, so metadata one fixture seeded would + /// otherwise answer another fixture's lookup and hide a missing arrangement. + /// + [SetUp] + public void SetUpFakeOrgTest() + { + SharedStaticCaches.Reset(); + Org = null; + Recorder = null; + } + + /// An org that supports no bulk messages - the MMSTEST2 shape. + protected ShuffleTestContext OnPrem() + { + Org = ShuffleTestContext.AsOnPrem(); + return Org; + } + + /// An org where every seeded entity supports every bulk message - the ImransDev shape. + protected ShuffleTestContext Online() + { + Org = ShuffleTestContext.AsOnline(); + return Org; + } + + /// A shuffler over the current org, with its events already captured. + protected Shuffler NewShuffler(bool stopOnError = false) + { + if (Org == null) + { + throw new InvalidOperationException("Build an org with OnPrem() or Online() first."); + } + var shuffler = Shuffler.CreateForTest(Org.Container, stopOnError); + Recorder = new ShuffleEventRecorder(Org.Logger); + Recorder.Attach(shuffler); + return shuffler; + } + + /// A record with a name, for fixtures that only care that it is distinguishable. + protected static Entity Record(string entityLogicalName, Guid id, string name = null) + { + var entity = new Entity(entityLogicalName, id); + entity["name"] = name ?? id.ToString(); + return entity; + } + + /// + /// A row as the target database holds it: like , but with the primary + /// id attribute populated. + /// + /// + /// Seeded rows need this and source records do not. Match queries always ask for the + /// primary id attribute (GetMatchingRecords seeds the column set with it), and a + /// real retrieve answers with it populated, so a row that omits it is less faithful than + /// one that carries it - and the fake org refuses to project an attribute a row does not + /// hold. + /// + protected static Entity Seeded(string entityLogicalName, Guid id, string name = null) + { + var entity = Record(entityLogicalName, id, name); + entity[entityLogicalName + "id"] = id; + return entity; + } + + /// Deterministic guids, so a failure message names the same record every run. + protected static Guid Id(int seed) + { + return new Guid(seed, 0, 0, new byte[] { 0, 0, 0, 0, 0, 0, 0, 0 }); + } + + /// Everything both channels saw plus the requests sent, for a failure message. + protected string DumpAll() + { + var log = Recorder == null ? "(no shuffler built)" : Recorder.Dump(); + var requests = Org == null ? "(no org built)" : string.Join(", ", Service.RequestNames); + var faults = Org == null || Org.Logger.Exceptions.Count == 0 + ? string.Empty + : Environment.NewLine + "--- exceptions ---" + Environment.NewLine + + string.Join(Environment.NewLine, Org.Logger.Exceptions); + return log + Environment.NewLine + "Requests: " + requests + faults; + } + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Helpers/FetchXmlConversion.cs b/tests/Xrm.Shuffle.Core.Tests/Helpers/FetchXmlConversion.cs new file mode 100644 index 0000000..015ce53 --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Helpers/FetchXmlConversion.cs @@ -0,0 +1,51 @@ +namespace Cinteros.Crm.Utils.Shuffle.Tests.Helpers +{ + using Microsoft.Crm.Sdk.Messages; + using Microsoft.Xrm.Sdk; + using Microsoft.Xrm.Sdk.Query; + + /// + /// Answers the one message the product only sends in a Debug build. + /// + /// + /// + /// Two log lines in ShuffleDataImport sit inside #if DEBUG and print the match + /// query as FetchXml, which ConvertToFetchXml obtains by sending a + /// QueryExpressionToFetchXmlRequest. FakeXrmEasy 1.x has no executor for it and throws, and + /// the scripted service throws for anything unscripted, so under Debug every fixture that + /// reaches a matched block died before the match query ran. Release compiled the lines out, + /// which is why the suite was green there and red in Visual Studio. + /// + /// + /// The product uses the returned string for nothing but the log line, so the answer here is + /// a token rather than a real conversion - modelling the platform FetchXml writer would add + /// a lot of surface for no assertion. The request is deliberately not recorded: keeping it + /// out of the request list is what makes a fixture see the same sequence of calls in both + /// configurations. + /// + /// + internal static class FetchXmlConversion + { + /// The untyped name the request arrives under. + internal const string MessageName = "QueryExpressionToFetchXml"; + + /// True when this request is the Debug-only conversion. + internal static bool IsConversion(OrganizationRequest request) + { + return request != null && request.RequestName == MessageName; + } + + /// A response carrying enough FetchXml to log, named after the query entity. + internal static OrganizationResponse Answer(OrganizationRequest request) + { + var query = request.Parameters.Contains("Query") + ? request.Parameters["Query"] as QueryExpression + : null; + var response = new QueryExpressionToFetchXmlResponse(); + response.Results["FetchXml"] = string.Format( + @"", + query == null ? "unknown" : query.EntityName); + return response; + } + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Helpers/RecordingLogger.cs b/tests/Xrm.Shuffle.Core.Tests/Helpers/RecordingLogger.cs new file mode 100644 index 0000000..d909672 --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Helpers/RecordingLogger.cs @@ -0,0 +1,99 @@ +namespace Cinteros.Crm.Utils.Shuffle.Tests.Helpers +{ + using System; + using System.Collections.Generic; + using System.Globalization; + using System.Linq; + using global::Xrm.Utils.Core.Common.Interfaces; + + /// + /// An that keeps everything in memory. + /// + /// + /// The only concrete container in the product, CintContainer, logs to + /// C:\Temp\<AssemblyName> through FileLogger - so tests must never use it. + /// This records instead of writing, and tracks section depth so a fixture can + /// assert that a section was opened and closed rather than left dangling. + /// + public class RecordingLogger : ILoggable + { + private readonly List messages = new List(); + private readonly List exceptions = new List(); + private readonly List sections = new List(); + + /// Every message logged, in order, already formatted. + public IReadOnlyList Messages => messages; + + /// Every exception logged, in order. + public IReadOnlyList Exceptions => exceptions; + + /// Names of the sections currently open, outermost first. + public IReadOnlyList OpenSections => sections; + + /// How deeply sections are nested right now. + public int SectionDepth => sections.Count; + + /// True once has been called. + public bool Closed { get; private set; } + + /// Text passed to , if any. + public string CloseText { get; private set; } + + public void CloseLog() + { + Closed = true; + } + + public void CloseLog(string closetext) + { + CloseText = closetext; + Closed = true; + } + + public void EndSection() + { + // Not an assertion: the product calls EndSection from finally blocks and can + // outdent past zero on an error path. Swallowing here keeps a fixture's real + // failure visible instead of masking it with a helper crash. + if (sections.Count > 0) + { + sections.RemoveAt(sections.Count - 1); + } + } + + public void Log(string message) + { + messages.Add(message); + } + + public void Log(Exception ex) + { + exceptions.Add(ex); + } + + public void Log(string message, params object[] arg) + { + // Matches the product's own formatting: Logger.Log(message, args) goes through + // string.Format, so a message containing stray braces throws there too. + messages.Add(arg == null || arg.Length == 0 + ? message + : string.Format(CultureInfo.InvariantCulture, message, arg)); + } + + public void StartSection(string name = null) + { + sections.Add(name); + } + + /// True if any logged message contains . + public bool Logged(string fragment) => + messages.Any(m => m != null && m.IndexOf(fragment, StringComparison.Ordinal) >= 0); + + /// How many logged messages contain . + public int CountLogged(string fragment) => + messages.Count(m => m != null && m.IndexOf(fragment, StringComparison.Ordinal) >= 0); + + /// All messages joined by newlines - for assertion failure output. + public string Dump() => string.Join(Environment.NewLine, messages); + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Helpers/RecordingOrganizationService.cs b/tests/Xrm.Shuffle.Core.Tests/Helpers/RecordingOrganizationService.cs new file mode 100644 index 0000000..0573238 --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Helpers/RecordingOrganizationService.cs @@ -0,0 +1,285 @@ +namespace Cinteros.Crm.Utils.Shuffle.Tests.Helpers +{ + using System; + using System.Collections.Generic; + using System.Linq; + using FakeXrmEasy; + using Microsoft.Xrm.Sdk; + using Microsoft.Xrm.Sdk.Messages; + using Microsoft.Xrm.Sdk.Query; + + /// + /// The fake org's own , wrapped so that a fixture can see + /// which requests the import actually sent, and so that the three bulk messages work at all. + /// + /// + /// + /// Two jobs, both of which have to happen at this layer. The first is recording: FakeXrmEasy + /// answers a request and forgets it, but the whole point of these fixtures is which rung of + /// the fallback chain the product landed on, so every call is kept in order. + /// + /// + /// The second is CreateMultiple, UpdateMultiple and UpsertMultiple. FakeXrmEasy 1.x ships no + /// executor for any of them - they postdate it - and the product sends them untyped, as + /// new OrganizationRequest("CreateMultiple") with a Targets collection. So they + /// are served here by fanning the targets out over the single-record messages the fake does + /// understand, and answering in the shape the product reads back: Ids for + /// CreateMultiple, nothing for UpdateMultiple, Results for UpsertMultiple. + /// + /// + /// Fanning out is a deliberate simplification, and it is worth being clear about what it + /// costs. A real CreateMultiple is one transaction: one bad row rolls the batch back. Here + /// the rows before the bad one are already written. Tests about that boundary belong in + /// Layer 1, where the service is scripted and can fault the batch as a unit; see + /// . What this class is for is routing - proving + /// that the block reached CreateMultiple at all, with the targets it should have carried. + /// + /// + public class RecordingOrganizationService : IOrganizationService + { + /// The messages this wrapper serves itself rather than passing to the fake. + private static readonly string[] BulkMessages = { "CreateMultiple", "UpdateMultiple", "UpsertMultiple" }; + + private readonly IOrganizationService inner; + private readonly List requests = new List(); + private readonly List created = new List(); + private readonly List updated = new List(); + private readonly List> deleted = new List>(); + private readonly List queries = new List(); + + private readonly Dictionary> throwOnce = + new Dictionary>(StringComparer.Ordinal); + private readonly Dictionary throwAlways = + new Dictionary(StringComparer.Ordinal); + + public RecordingOrganizationService(XrmFakedContext faked) + { + if (faked == null) + { + throw new ArgumentNullException("faked"); + } + inner = faked.GetOrganizationService(); + } + + /// Every request that reached , in order. + public IReadOnlyList Requests + { + get { return requests; } + } + + /// Entities passed to the individual path. + public IReadOnlyList Created + { + get { return created; } + } + + /// Entities passed to the individual path. + public IReadOnlyList Updated + { + get { return updated; } + } + + /// Records passed to the individual path. + public IReadOnlyList> Deleted + { + get { return deleted; } + } + + /// + /// Every query, in order - capability probes and match retrievals alike. Queries do not + /// go through Execute, so this is the only place they are visible. + /// + public IReadOnlyList Queries + { + get { return queries; } + } + + /// The request names seen, in order - the routing assertion most fixtures make. + public IReadOnlyList RequestNames + { + get { return requests.Select(r => r.RequestName).ToList(); } + } + + /// How many requests named were executed. + public int CountOf(string messageName) + { + return requests.Count(r => string.Equals(r.RequestName, messageName, StringComparison.Ordinal)); + } + + /// The requests named , in order. + public IReadOnlyList RequestsNamed(string messageName) + { + return requests.Where(r => string.Equals(r.RequestName, messageName, StringComparison.Ordinal)).ToList(); + } + + /// The targets a bulk request carried, or an empty list if it carried none. + public static IReadOnlyList TargetsOf(OrganizationRequest request) + { + var targets = request.Parameters.Contains("Targets") + ? request.Parameters["Targets"] as EntityCollection + : null; + return targets == null ? new List() : targets.Entities.ToList(); + } + + /// + /// Throws the next time is + /// executed, then lets the message through. This is how the fallback chain is reached: + /// the bulk message fails once and the product is expected to drop a rung and succeed. + /// + public RecordingOrganizationService ThrowOnce(string messageName, Exception exception) + { + Queue queue; + if (!throwOnce.TryGetValue(messageName, out queue)) + { + queue = new Queue(); + throwOnce[messageName] = queue; + } + queue.Enqueue(exception); + return this; + } + + /// Throws every time is executed. + public RecordingOrganizationService ThrowAlways(string messageName, Exception exception) + { + throwAlways[messageName] = exception; + return this; + } + + #region IOrganizationService + + public Guid Create(Entity entity) + { + created.Add(entity); + Record("Create", "Target", entity); + Fault("Create"); + return inner.Create(entity); + } + + public void Update(Entity entity) + { + updated.Add(entity); + Record("Update", "Target", entity); + Fault("Update"); + inner.Update(entity); + } + + public void Delete(string entityName, Guid id) + { + deleted.Add(Tuple.Create(entityName, id)); + Record("Delete", "Target", new EntityReference(entityName, id)); + Fault("Delete"); + inner.Delete(entityName, id); + } + + public Entity Retrieve(string entityName, Guid id, ColumnSet columnSet) + { + Record("Retrieve", "Target", new EntityReference(entityName, id)); + Fault("Retrieve"); + return inner.Retrieve(entityName, id, columnSet); + } + + public EntityCollection RetrieveMultiple(QueryBase query) + { + queries.Add(query); + return inner.RetrieveMultiple(query); + } + + public OrganizationResponse Execute(OrganizationRequest request) + { + if (FetchXmlConversion.IsConversion(request)) + { + return FetchXmlConversion.Answer(request); + } + + requests.Add(request); + Fault(request.RequestName); + + return BulkMessages.Contains(request.RequestName, StringComparer.Ordinal) + ? ExecuteBulk(request) + : inner.Execute(request); + } + + public void Associate(string entityName, Guid entityId, Relationship relationship, EntityReferenceCollection relatedEntities) + { + inner.Associate(entityName, entityId, relationship, relatedEntities); + } + + public void Disassociate(string entityName, Guid entityId, Relationship relationship, EntityReferenceCollection relatedEntities) + { + inner.Disassociate(entityName, entityId, relationship, relatedEntities); + } + + #endregion + + /// + /// Serves one of the three bulk messages by fanning its targets out over the + /// single-record messages the fake understands. See the remarks on this class for why + /// that is not the same thing as a real bulk request. + /// + private OrganizationResponse ExecuteBulk(OrganizationRequest request) + { + var targets = TargetsOf(request); + var response = new OrganizationResponse { ResponseName = request.RequestName }; + + switch (request.RequestName) + { + case "CreateMultiple": + var ids = new Guid[targets.Count]; + for (var i = 0; i < targets.Count; i++) + { + ids[i] = inner.Create(targets[i]); + } + response.Results["Ids"] = ids; + break; + + case "UpdateMultiple": + foreach (var target in targets) + { + inner.Update(target); + } + break; + + case "UpsertMultiple": + var results = new UpsertResponse[targets.Count]; + for (var i = 0; i < targets.Count; i++) + { + results[i] = (UpsertResponse)inner.Execute(new UpsertRequest { Target = targets[i] }); + } + response.Results["Results"] = results; + break; + + default: + throw new InvalidOperationException(request.RequestName + " is not a bulk message."); + } + + return response; + } + + /// Throws whatever the fixture armed for this message, if anything. + private void Fault(string messageName) + { + Exception always; + if (throwAlways.TryGetValue(messageName, out always)) + { + throw always; + } + + Queue pending; + if (throwOnce.TryGetValue(messageName, out pending) && pending.Count > 0) + { + throw pending.Dequeue(); + } + } + + /// + /// Logs an individual operation as a request, so RequestNames and CountOf see the + /// individual rungs of the fallback chain the same way they see the batched ones. + /// + private void Record(string messageName, string parameterName, object target) + { + var request = new OrganizationRequest(messageName); + request[parameterName] = target; + requests.Add(request); + } + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Helpers/ScriptedOrganizationService.cs b/tests/Xrm.Shuffle.Core.Tests/Helpers/ScriptedOrganizationService.cs new file mode 100644 index 0000000..196974a --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Helpers/ScriptedOrganizationService.cs @@ -0,0 +1,346 @@ +namespace Cinteros.Crm.Utils.Shuffle.Tests.Helpers +{ + using System; + using System.Collections.Generic; + using System.Linq; + using Microsoft.Xrm.Sdk; + using Microsoft.Xrm.Sdk.Query; + + /// + /// An whose every answer a fixture writes itself. + /// + /// + /// + /// FakeXrmEasy 1.x answers plausibly, which is the wrong tool for the batch paths: the + /// interesting cases are implausible. A batch response with fewer items than requests, an + /// out-of-order RequestIndex, a bulk message that faults with "not implemented" on the + /// second call but not the first - none of those can be asked of a fake that models a + /// real platform. So this service scripts responses instead of simulating an org, and + /// records every request so a fixture can assert which path the product actually took. + /// + /// + /// Anything a fixture has not scripted throws. Silence would let a test pass while the + /// product quietly took a different route, which is the one failure mode these tests + /// exist to catch. + /// + /// + public class ScriptedOrganizationService : IOrganizationService + { + private readonly List requests = new List(); + private readonly List created = new List(); + private readonly List updated = new List(); + private readonly List> deleted = new List>(); + + private readonly Dictionary> byMessage = + new Dictionary>(StringComparer.Ordinal); + + private readonly Dictionary> throwOnce = + new Dictionary>(StringComparer.Ordinal); + + private readonly Dictionary> supportedBulkMessages = + new Dictionary>(StringComparer.OrdinalIgnoreCase); + + private readonly List> probes = new List>(); + + private Exception probeFailure; + private Func onCreate; + private Action onUpdate; + private Func onRetrieveMultiple; + + /// Every request that reached , in order. + public IReadOnlyList Requests => requests; + + /// Entities passed to the individual path. + public IReadOnlyList Created => created; + + /// Entities passed to the individual path. + public IReadOnlyList Updated => updated; + + /// Records passed to the individual path. + public IReadOnlyList> Deleted => deleted; + + /// + /// Every sdkmessagefilter capability probe, as (entity, message). RetrieveMultiple does + /// not go through Requests, and the probe is cached per entity and message, so this is + /// what a fixture counts to show the cache is doing its job. + /// + public IReadOnlyList> Probes => probes; + + /// The request names seen, in order - the routing assertion most fixtures make. + public IReadOnlyList RequestNames => requests.Select(r => r.RequestName).ToList(); + + /// How many requests named were executed. + public int CountOf(string messageName) => + requests.Count(r => string.Equals(r.RequestName, messageName, StringComparison.Ordinal)); + + /// The requests named , in order. + public IReadOnlyList RequestsNamed(string messageName) => + requests.Where(r => string.Equals(r.RequestName, messageName, StringComparison.Ordinal)).ToList(); + + #region scripting + + /// + /// Answers with . + /// + /// + /// Keyed on RequestName rather than on a request type, because the bulk messages are + /// built as untyped OrganizationRequest("CreateMultiple") - there is no + /// CreateMultipleRequest in the 9.0 SDK assemblies this repo builds against. + /// + public ScriptedOrganizationService OnMessage(string messageName, Func handler) + { + byMessage[messageName] = handler; + return this; + } + + /// Answers with a fixed response. + public ScriptedOrganizationService OnMessage(string messageName, OrganizationResponse response) + { + return OnMessage(messageName, _ => response); + } + + /// + /// Answers with the queued responses, one per call, in order. + /// + public ScriptedOrganizationService OnMessageSequence(string messageName, params OrganizationResponse[] responses) + { + var queue = new Queue(responses); + var scripted = responses.Length; + return OnMessage(messageName, _ => + { + if (queue.Count == 0) + { + throw new InvalidOperationException(string.Format( + "{0} was executed more times than the fixture scripted ({1}).", messageName, scripted)); + } + return queue.Dequeue(); + }); + } + + /// + /// Throws the next time is + /// executed, then falls through to whatever else is scripted. + /// + /// + /// This is how the fallback chain gets exercised: the bulk message throws + /// "not implemented" once, and the product is expected to fall back and succeed on + /// the next rung rather than give up. + /// + public ScriptedOrganizationService ThrowOnce(string messageName, Exception exception) + { + Queue queue; + if (!throwOnce.TryGetValue(messageName, out queue)) + { + queue = new Queue(); + throwOnce[messageName] = queue; + } + queue.Enqueue(exception); + return this; + } + + /// + /// Declares that supports the named bulk + /// messages, so the capability probe answers yes for exactly those. + /// + public ScriptedOrganizationService SupportsBulkMessage(string entityLogicalName, params string[] messageNames) + { + HashSet set; + if (!supportedBulkMessages.TryGetValue(entityLogicalName, out set)) + { + set = new HashSet(StringComparer.OrdinalIgnoreCase); + supportedBulkMessages[entityLogicalName] = set; + } + foreach (var name in messageNames) + { + set.Add(name); + } + return this; + } + + /// + /// Makes the capability probe throw. The product catches that and caches a no, which + /// is the difference between an org that cannot answer and one that answers no. + /// + public ScriptedOrganizationService FailTheCapabilityProbe(Exception exception) + { + probeFailure = exception; + return this; + } + + /// Assigns ids to individual creates. Defaults to a fresh guid each time. + public ScriptedOrganizationService OnCreate(Func handler) + { + onCreate = handler; + return this; + } + + /// Observes individual updates. Defaults to accepting them. + public ScriptedOrganizationService OnUpdate(Action handler) + { + onUpdate = handler; + return this; + } + + /// + /// Answers any RetrieveMultiple the capability probe did not claim - match queries, + /// mostly. Returning an empty collection is the common case and has to be explicit. + /// + public ScriptedOrganizationService OnRetrieveMultiple(Func handler) + { + onRetrieveMultiple = handler; + return this; + } + + #endregion + + #region IOrganizationService + + public Guid Create(Entity entity) + { + created.Add(entity); + Record("Create", "Target", entity); + return onCreate != null ? onCreate(entity) : Guid.NewGuid(); + } + + public void Update(Entity entity) + { + updated.Add(entity); + Record("Update", "Target", entity); + if (onUpdate != null) + { + onUpdate(entity); + } + } + + public void Delete(string entityName, Guid id) + { + deleted.Add(Tuple.Create(entityName, id)); + Record("Delete", "Target", new EntityReference(entityName, id)); + } + + /// + /// Logs an individual operation as a request, so that RequestNames and CountOf see the + /// individual rungs of the fallback chain the same way they see the batched ones. + /// + private void Record(string messageName, string parameterName, object target) + { + var request = new OrganizationRequest(messageName); + request[parameterName] = target; + requests.Add(request); + } + + public OrganizationResponse Execute(OrganizationRequest request) + { + if (FetchXmlConversion.IsConversion(request)) + { + return FetchXmlConversion.Answer(request); + } + + requests.Add(request); + + Queue pending; + if (throwOnce.TryGetValue(request.RequestName, out pending) && pending.Count > 0) + { + throw pending.Dequeue(); + } + + Func handler; + if (byMessage.TryGetValue(request.RequestName, out handler)) + { + return handler(request); + } + + throw new InvalidOperationException(string.Format( + "The fixture did not script {0}. Requests so far: {1}.", + request.RequestName, + string.Join(", ", RequestNames))); + } + + public EntityCollection RetrieveMultiple(QueryBase query) + { + var probe = AsBulkCapabilityProbe(query); + if (probe != null) + { + probes.Add(probe); + if (probeFailure != null) + { + throw probeFailure; + } + + HashSet supported; + var yes = supportedBulkMessages.TryGetValue(probe.Item1, out supported) + && supported.Contains(probe.Item2); + var result = new EntityCollection { EntityName = "sdkmessagefilter" }; + if (yes) + { + result.Entities.Add(new Entity("sdkmessagefilter", Guid.NewGuid())); + } + return result; + } + + if (onRetrieveMultiple != null) + { + return onRetrieveMultiple(query); + } + + throw new InvalidOperationException( + "The fixture did not script RetrieveMultiple, and the query is not a bulk capability probe."); + } + + public Entity Retrieve(string entityName, Guid id, ColumnSet columnSet) + { + throw new NotSupportedException( + "Retrieve is not scripted. The import paths under test use RetrieveMultiple; " + + "reaching this means the product took an unexpected route."); + } + + public void Associate(string entityName, Guid entityId, Relationship relationship, EntityReferenceCollection relatedEntities) + { + throw new NotSupportedException("Associate is not scripted."); + } + + public void Disassociate(string entityName, Guid entityId, Relationship relationship, EntityReferenceCollection relatedEntities) + { + throw new NotSupportedException("Disassociate is not scripted."); + } + + #endregion + + /// + /// Recognises the sdkmessagefilter query IsBulkMessageSupported builds, returning + /// entity logical name and message name when it matches, and null otherwise. + /// + /// + /// Note which value the entity condition carries: primaryobjecttypecode really holds a + /// numeric entity type code, but the product queries it with the logical name string. + /// This helper matches the product, not the platform, deliberately - see the marker + /// test Capability_query_uses_the_logical_name_not_the_entity_type_code. + /// + private static Tuple AsBulkCapabilityProbe(QueryBase query) + { + var expression = query as QueryExpression; + if (expression == null || expression.EntityName != "sdkmessagefilter") + { + return null; + } + + var entityCondition = expression.Criteria.Conditions + .FirstOrDefault(c => c.AttributeName == "primaryobjecttypecode"); + var link = expression.LinkEntities.FirstOrDefault(l => l.LinkToEntityName == "sdkmessage"); + if (entityCondition == null || link == null) + { + return null; + } + + var messageCondition = link.LinkCriteria.Conditions.FirstOrDefault(c => c.AttributeName == "name"); + if (messageCondition == null) + { + return null; + } + + return Tuple.Create( + Convert.ToString(entityCondition.Values.FirstOrDefault()), + Convert.ToString(messageCondition.Values.FirstOrDefault())); + } + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Helpers/SharedStaticCaches.cs b/tests/Xrm.Shuffle.Core.Tests/Helpers/SharedStaticCaches.cs new file mode 100644 index 0000000..4ffe73f --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Helpers/SharedStaticCaches.cs @@ -0,0 +1,100 @@ +namespace Cinteros.Crm.Utils.Shuffle.Tests.Helpers +{ + using System; + using System.Collections.Concurrent; + using System.Collections.Generic; + using System.Linq; + using System.Reflection; + using System.Runtime.Caching; + + /// + /// Clears the process-wide caches in Xrm.Utils.Core between tests. + /// + /// + /// + /// Three static caches outlive a fixture and would otherwise carry answers from one test + /// into the next: the primary-id and primary-name attribute lookups, which exist twice + /// (once on the extension methods, once on the fluent API), and the metadata MemoryCache + /// the container extensions keep with a five-minute sliding expiry. All are private, so + /// reflection is the only way in; all are readonly, so the MemoryCache is drained + /// key by key rather than replaced. + /// + /// + /// Without this, a test that seeds a fake org with one entity shape can be answered from + /// a cache another test populated, and the result depends on execution order. With it, + /// the order stops mattering. + /// + /// + public static class SharedStaticCaches + { + private const BindingFlags PrivateStatic = BindingFlags.Static | BindingFlags.NonPublic; + + private static readonly Lazy> AttributeCaches = + new Lazy>(ResolveAttributeCaches); + + private static readonly Lazy MetadataCache = new Lazy(ResolveMetadataCache); + + /// Empties every cache. Call from [SetUp]. + public static void Reset() + { + foreach (var field in AttributeCaches.Value) + { + var dictionary = (ConcurrentDictionary)field.GetValue(null); + dictionary.Clear(); + } + + var cache = (MemoryCache)MetadataCache.Value.GetValue(null); + // MemoryCache has no Clear, and the field is readonly so it cannot be replaced. + // Snapshot the keys first: removing while enumerating the cache itself is not safe. + foreach (var key in cache.Select(entry => entry.Key).ToList()) + { + cache.Remove(key); + } + } + + /// + /// Proves every cache was found. Call from [OneTimeSetUp]. + /// + /// + /// Load-bearing: reflection by name fails silently if the submodule renames a type or + /// field, and a silent failure here turns Reset into a no-op, which shows up much + /// later as a test that only fails when the suite runs in a particular order. + /// + public static void AssertResolved() + { + if (AttributeCaches.Value.Count != 4) + { + throw new InvalidOperationException( + "Expected 4 attribute-name caches in Xrm.Utils.Core, found " + AttributeCaches.Value.Count + + ". A type or field was renamed, and clearing them has become a no-op."); + } + + if (MetadataCache.Value == null) + { + throw new InvalidOperationException( + "The metadata MemoryCache field was not found in ContainerExtensions."); + } + } + + private static IReadOnlyList ResolveAttributeCaches() + { + var names = new[] { "PrimaryIdAttributes", "PrimaryNameAttributes" }; + var types = new[] + { + typeof(global::Xrm.Utils.Core.Common.Extensions.EntityExtensions), + typeof(global::Xrm.Utils.Core.Common.Fluent.Entity.OperationsSet2) + }; + + return types + .SelectMany(type => names.Select(name => type.GetField(name, PrivateStatic))) + .Where(field => field != null) + .ToList(); + } + + private static FieldInfo ResolveMetadataCache() + { + return typeof(global::Xrm.Utils.Core.Common.Extensions.ContainerExtensions) + .GetField("cache", PrivateStatic); + } + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Helpers/ShuffleEventRecorder.cs b/tests/Xrm.Shuffle.Core.Tests/Helpers/ShuffleEventRecorder.cs new file mode 100644 index 0000000..a2b3c69 --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Helpers/ShuffleEventRecorder.cs @@ -0,0 +1,125 @@ +namespace Cinteros.Crm.Utils.Shuffle.Tests.Helpers +{ + using System; + using System.Collections.Generic; + using System.Linq; + using NUnit.Framework; + + /// + /// Captures the events a raises, alongside what it logged. + /// + /// + /// + /// The product talks to the outside world through two channels and neither is a superset + /// of the other, so a fixture that watches only one can miss a regression entirely: + /// + /// + /// SendText logs through the container, but only when the formatted message is + /// longer than one character - so a bare newline reaches the event stream and never the + /// log. + /// SendStatus raises an event whose Message is null, carrying only block and record + /// counters - so progress reporting is invisible to the log. + /// + /// + /// Both channels are therefore recorded, and the assertion helpers name which one they + /// look at: Logged for the container, Sent for event messages, Raised for events of any + /// kind including the message-less status ones. + /// + /// + public class ShuffleEventRecorder + { + private readonly List events = new List(); + + public ShuffleEventRecorder(RecordingLogger logger) + { + Logger = logger; + } + + /// The log side of the pair. + public RecordingLogger Logger { get; } + + /// Every event raised, in order. + public IReadOnlyList Events => events; + + /// Event messages, in order, with the message-less status events dropped. + public IReadOnlyList SentMessages => + events.Where(e => e.Message != null).Select(e => e.Message).ToList(); + + /// Attaches to a shuffler. Safe to call once per instance. + public void Attach(Shuffler shuffler) + { + shuffler.RaiseShuffleEvent += OnShuffleEvent; + } + + public void OnShuffleEvent(object sender, ShuffleEventArgs args) + { + events.Add(args); + } + + #region log channel + + /// Asserts the container logged something containing . + public void AssertLogged(string fragment) + { + Assert.That(Logger.Logged(fragment), Is.True, + "Expected a log message containing \"{0}\". Logged:{1}{2}", + fragment, Environment.NewLine, Logger.Dump()); + } + + /// Asserts nothing logged contains . + public void AssertNeverLogged(string fragment) + { + Assert.That(Logger.Logged(fragment), Is.False, + "Expected no log message containing \"{0}\", but found {1}. Logged:{2}{3}", + fragment, Logger.CountLogged(fragment), Environment.NewLine, Logger.Dump()); + } + + /// Asserts exactly log messages contain the fragment. + public void AssertLoggedTimes(string fragment, int times) + { + Assert.That(Logger.CountLogged(fragment), Is.EqualTo(times), + "Expected \"{0}\" in {1} log messages. Logged:{2}{3}", + fragment, times, Environment.NewLine, Logger.Dump()); + } + + #endregion + + #region event channel + + /// Asserts an event message contains . + public void AssertSent(string fragment) + { + Assert.That(SentMessages.Any(m => m.IndexOf(fragment, StringComparison.Ordinal) >= 0), Is.True, + "Expected a raised event whose message contains \"{0}\". Sent:{1}{2}", + fragment, Environment.NewLine, string.Join(Environment.NewLine, SentMessages)); + } + + /// Asserts no event message contains . + public void AssertNeverSent(string fragment) + { + Assert.That(SentMessages.Any(m => m.IndexOf(fragment, StringComparison.Ordinal) >= 0), Is.False, + "Expected no raised event whose message contains \"{0}\". Sent:{1}{2}", + fragment, Environment.NewLine, string.Join(Environment.NewLine, SentMessages)); + } + + /// Asserts at least one event was raised matching . + public void AssertRaised(Func predicate, string because) + { + Assert.That(events.Any(predicate), Is.True, + "Expected an event: {0}. {1} events were raised, {2} of them with a message.", + because, events.Count, SentMessages.Count); + } + + #endregion + + /// Everything both channels saw - for a failure message worth reading. + public string Dump() + { + return string.Concat( + "--- logged ---", Environment.NewLine, + Logger.Dump(), Environment.NewLine, + "--- sent ---", Environment.NewLine, + string.Join(Environment.NewLine, SentMessages)); + } + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Helpers/ShuffleTestBase.cs b/tests/Xrm.Shuffle.Core.Tests/Helpers/ShuffleTestBase.cs new file mode 100644 index 0000000..e25051f --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Helpers/ShuffleTestBase.cs @@ -0,0 +1,80 @@ +using System; +using Microsoft.Xrm.Sdk; +using NUnit.Framework; + +namespace Cinteros.Crm.Utils.Shuffle.Tests.Helpers +{ + /// + /// Wires the container, the scripted service and the event recorder, and clears the static + /// caches that would otherwise leak between fixtures. + /// + /// + /// The caches are the reason this base class exists. Four attribute-name dictionaries and one + /// MemoryCache in Xrm.Utils.Core.Common are process-wide statics, so a fixture that seeds + /// metadata for account would silently satisfy the next fixture's lookup and hide a + /// missing arrangement. runs before every test. + /// + public abstract class ShuffleTestBase + { + /// The scripted service every fixture arranges against. + protected ScriptedOrganizationService Service { get; private set; } + + /// The container handed to the code under test. + protected TestExecutionContainer Container { get; private set; } + + /// Both output channels — the log and the ShuffleEvent stream. + protected ShuffleEventRecorder Recorder { get; private set; } + + /// Fails the whole fixture early if the cache reset has stopped finding its fields. + [OneTimeSetUp] + public void AssertCachesAreReachable() + { + SharedStaticCaches.AssertResolved(); + } + + /// Fresh doubles and empty caches before every test. + [SetUp] + public void SetUpShuffleTest() + { + SharedStaticCaches.Reset(); + Service = new ScriptedOrganizationService(); + var logger = new RecordingLogger(); + Container = new TestExecutionContainer(Service, logger); + Recorder = new ShuffleEventRecorder(logger); + OnSetUp(); + } + + /// Override for per-fixture arrangement that needs the doubles in place. + protected virtual void OnSetUp() + { + } + + /// Builds a Shuffler over the test container, with its events already captured. + protected Shuffler NewShuffler(bool stopOnError = false) + { + var shuffler = Shuffler.CreateForTest(Container, stopOnError); + Recorder.Attach(shuffler); + return shuffler; + } + + /// A record with a name, for fixtures that only care that it is distinguishable. + protected static Entity Record(string entityLogicalName, Guid id, string name = null) + { + var entity = new Entity(entityLogicalName, id); + entity["name"] = name ?? id.ToString(); + return entity; + } + + /// Deterministic guids, so a failure message names the same record every run. + protected static Guid Id(int seed) + { + return new Guid(seed, 0, 0, new byte[] { 0, 0, 0, 0, 0, 0, 0, 0 }); + } + + /// Everything both channels saw, for pasting into a failure message. + protected string DumpAll() + { + return Recorder.Dump() + Environment.NewLine + "Requests: " + string.Join(", ", Service.RequestNames); + } + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Helpers/ShuffleTestContext.cs b/tests/Xrm.Shuffle.Core.Tests/Helpers/ShuffleTestContext.cs new file mode 100644 index 0000000..bd8008f --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Helpers/ShuffleTestContext.cs @@ -0,0 +1,270 @@ +namespace Cinteros.Crm.Utils.Shuffle.Tests.Helpers +{ + using System; + using System.Collections.Generic; + using System.Linq; + using FakeXrmEasy; + using Microsoft.Xrm.Sdk; + using Microsoft.Xrm.Sdk.Metadata; + + /// + /// A fake Dataverse organization, shaped like one of the two estates this feature runs + /// against, wrapped in the execution container the import code expects. + /// + /// + /// + /// The distinction that matters is capability. answers the + /// sdkmessagefilter probe for every entity it is given, so blocks route onto + /// CreateMultiple and friends; answers nothing, so the same + /// block falls back to ExecuteMultiple. Those are the two answers the real estates + /// give - ImransDev reports bulk support, MMSTEST2 does not. + /// + /// + /// The probe query filters primaryobjecttypecode by the entity's logical name + /// (ShuffleDataImport.cs, IsBulkMessageSupported), although on a real platform + /// that column holds a numeric entity type code. The seed here matches the product rather + /// than the platform, deliberately, and a test in CapabilityDetectionTests marks the + /// spot so a fix cannot land silently. + /// + /// + /// Nothing may be seeded after the container has been handed out: FakeXrmEasy takes its + /// data in one Initialize call, so a fixture that seeded late would be asserting + /// against an org that does not contain what it just added. + /// + /// + public class ShuffleTestContext + { + /// The messages the product probes for, in the order it tries them. + public static readonly string[] BulkMessages = { "CreateMultiple", "UpdateMultiple", "UpsertMultiple", "Upsert" }; + + private readonly XrmFakedContext faked = new XrmFakedContext(); + private readonly List seed = new List(); + private readonly Dictionary metadata = + new Dictionary(StringComparer.OrdinalIgnoreCase); + private readonly Dictionary> extraAttributes = + new Dictionary>(StringComparer.OrdinalIgnoreCase); + private readonly List> supported = + new List>(); + private readonly Dictionary messageIds = + new Dictionary(StringComparer.OrdinalIgnoreCase); + private readonly bool bulkByDefault; + private RecordingOrganizationService service; + private TestExecutionContainer container; + + private ShuffleTestContext(bool bulkByDefault) + { + this.bulkByDefault = bulkByDefault; + faked.ValidateReferences = false; + } + + /// An estate with no bulk messages at all - the MMSTEST2 shape. + public static ShuffleTestContext AsOnPrem() + { + return new ShuffleTestContext(false); + } + + /// An estate where every seeded entity supports every bulk message - the ImransDev shape. + public static ShuffleTestContext AsOnline() + { + return new ShuffleTestContext(true); + } + + /// Seeds rows. Their logical names also get default metadata if none was given. + public ShuffleTestContext WithEntity(params Entity[] entities) + { + RefuseIfBuilt(); + foreach (var entity in entities) + { + seed.Add(entity); + WithMetadata(entity.LogicalName); + } + return this; + } + + /// + /// Declares an entity's metadata: the primary id and name attributes, which are the only + /// two the import path reads, plus the attribute list the fake org validates queries + /// against. + /// + /// + /// The attribute list is not optional. Once an entity has metadata, FakeXrmEasy answers + /// any query naming an attribute that metadata does not declare with "The attribute X + /// does not exist on this entity" - even when every row carries it. The list is assembled + /// in EnsureBuilt from the seeded rows plus whatever + /// adds, so an attribute only the source records carry has + /// to be declared. + /// + public ShuffleTestContext WithMetadata(string logicalName, string primaryIdAttribute = null, string primaryNameAttribute = "name") + { + RefuseIfBuilt(); + if (metadata.ContainsKey(logicalName)) + { + return this; + } + var entity = new EntityMetadata { LogicalName = logicalName }; + SetSealed(entity, "PrimaryIdAttribute", primaryIdAttribute ?? logicalName + "id"); + SetSealed(entity, "PrimaryNameAttribute", primaryNameAttribute); + metadata[logicalName] = entity; + if (bulkByDefault) + { + SupportsMessage(logicalName, BulkMessages); + } + return this; + } + + /// + /// Declares attributes beyond the ones the seeded rows carry - an attribute that only the + /// source records hold, for one, since a query naming it still has to pass validation. + /// + public ShuffleTestContext WithAttributes(string logicalName, params string[] attributes) + { + RefuseIfBuilt(); + WithMetadata(logicalName); + HashSet declared; + if (!extraAttributes.TryGetValue(logicalName, out declared)) + { + declared = new HashSet(StringComparer.OrdinalIgnoreCase); + extraAttributes[logicalName] = declared; + } + foreach (var attribute in attributes) + { + declared.Add(attribute); + } + return this; + } + + /// Answers the capability probe yes for these messages on this entity, and no for the rest. + public ShuffleTestContext SupportsMessage(string entityLogicalName, params string[] messages) + { + RefuseIfBuilt(); + foreach (var message in messages) + { + supported.Add(new KeyValuePair(entityLogicalName, message)); + } + return this; + } + + /// The container to hand to the shuffler. + public TestExecutionContainer Container + { + get { EnsureBuilt(); return container; } + } + + /// The service the container wraps, for asserting what was actually sent. + public RecordingOrganizationService Service + { + get { EnsureBuilt(); return service; } + } + + /// The log, for asserting the routing decision the product announced. + public RecordingLogger Logger + { + get { return Container.Recorder; } + } + + /// The fake org itself, for tests that need to reach past the container. + public XrmFakedContext Faked + { + get { EnsureBuilt(); return faked; } + } + + /// Every row of one entity, as the fake org holds it now. + public List Rows(string logicalName) + { + EnsureBuilt(); + return faked.CreateQuery(logicalName).ToList(); + } + + /// One row, or null. Reads through to the fake org, so it sees writes the test made. + public Entity Row(string logicalName, Guid id) + { + return Rows(logicalName).FirstOrDefault(e => e.Id == id); + } + + private void EnsureBuilt() + { + if (container != null) + { + return; + } + var rows = new List(seed); + rows.AddRange(CapabilityRows()); + DeclareAttributes(); + faked.InitializeMetadata(metadata.Values); + faked.Initialize(rows); + service = new RecordingOrganizationService(faked); + container = new TestExecutionContainer(service); + } + + private void RefuseIfBuilt() + { + if (container != null) + { + throw new InvalidOperationException( + "Seed the fake org before asking for the container - FakeXrmEasy takes its data in one Initialize call."); + } + } + + private IEnumerable CapabilityRows() + { + var rows = new List(); + foreach (var pair in supported) + { + Guid messageId; + if (!messageIds.TryGetValue(pair.Value, out messageId)) + { + messageId = Guid.NewGuid(); + messageIds[pair.Value] = messageId; + var sdkmessage = new Entity("sdkmessage", messageId); + sdkmessage["name"] = pair.Value; + rows.Add(sdkmessage); + } + var filter = new Entity("sdkmessagefilter", Guid.NewGuid()); + // The logical name, not the numeric type code - see the remarks on this class. + filter["primaryobjecttypecode"] = pair.Key; + filter["sdkmessageid"] = new EntityReference("sdkmessage", messageId); + rows.Add(filter); + } + return rows; + } + + /// + /// Gives every declared entity the attribute list its queries will be validated against: + /// the primary id and name, everything the seeded rows carry, and anything a fixture + /// declared on top. + /// + private void DeclareAttributes() + { + foreach (var pair in metadata) + { + var names = new HashSet(StringComparer.OrdinalIgnoreCase) + { + pair.Value.PrimaryIdAttribute, + pair.Value.PrimaryNameAttribute, + }; + foreach (var row in seed.Where(e => string.Equals(e.LogicalName, pair.Key, StringComparison.OrdinalIgnoreCase))) + { + foreach (var key in row.Attributes.Keys) + { + names.Add(key); + } + } + HashSet extras; + if (extraAttributes.TryGetValue(pair.Key, out extras)) + { + names.UnionWith(extras); + } + var attributes = names + .Select(name => (AttributeMetadata)new StringAttributeMetadata { LogicalName = name }) + .ToArray(); + SetSealed(pair.Value, "Attributes", attributes); + } + } + + private static void SetSealed(object target, string property, object value) + { + var setter = target.GetType().GetProperty(property).GetSetMethod(true); + setter.Invoke(target, new[] { value }); + } + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Helpers/ShufflerTestShim.cs b/tests/Xrm.Shuffle.Core.Tests/Helpers/ShufflerTestShim.cs new file mode 100644 index 0000000..c836e40 --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Helpers/ShufflerTestShim.cs @@ -0,0 +1,476 @@ +namespace Cinteros.Crm.Utils.Shuffle +{ + using System; + using System.Collections.Generic; + using System.Linq; + using Microsoft.Xrm.Sdk; + using Xrm.Utils.Core.Common.Interfaces; + + /// + /// Test-only surface over the private batch machinery in ShuffleDataImport. + /// + /// + /// + /// Shuffler is partial, so this file adds to the same class rather than reaching in from + /// outside. That matters for two reasons: the flush methods are private, and the batch + /// item types are private nested structs, which no external fixture can even name. A + /// wrapper type declared here can name them, because it is nested in Shuffler too. + /// + /// + /// The alternative - reflection, or InternalsVisibleTo on a shared project with no + /// assembly of its own - would either break silently on a rename or not work at all. + /// This costs one file that ships with the tests and never runs in production. + /// + /// + /// This file is deliberately mechanical: it unwraps arguments, calls the private member, + /// and wraps the result. Any logic here would be logic the tests are not testing. + /// + /// + public partial class Shuffler + { + /// + /// A Shuffler ready to have a flush method called on it directly. + /// + /// + /// The constructor only stores the container. guidmap, stoponerror and timeout are + /// set later, inside ImportToCRM - so a shuffler that has not run an import has a + /// null guidmap, and calling a flush method on it throws a NullReferenceException + /// that says nothing about the test. This performs the same initialisation for a + /// definition-less run. + /// + public static Shuffler CreateForTest(IExecutionContainer container, bool stopOnError = false) + { + var shuffler = new Shuffler(container); + shuffler.guidmap = new Dictionary(); + shuffler.stoponerror = stopOnError; + shuffler.timeout = -1; + return shuffler; + } + + /// The guid map, so a fixture can seed and inspect the remapping. + public Dictionary TestGuidMap => guidmap; + + /// + /// The record label StopOnBatchError stamps when it decides to abort. + /// + /// + /// Null until a batch error is recorded, which is the signal the import block reads + /// to decide whether to throw - so asserting on it is asserting on stop-on-error. + /// + public string TestBatchFailureLabel => batchFailureLabel; + + /// Whether this instance was created with stop-on-error set. + public bool TestStopOnError => stoponerror; + + /// What a flush left behind: the counters and the references it collected. + public class BatchOutcome + { + /// Records the flush counted as created. + public int Created; + + /// Records the flush counted as updated. + public int Updated; + + /// Records the flush counted as deleted. + public int Deleted; + + /// Records the flush counted as failed. + public int Failed; + + /// References the flush collected for the created records. + public EntityReferenceCollection References = new EntityReferenceCollection(); + + /// Created plus updated plus failed - every record the flush accounted for. + public int Accounted => Created + Updated + Failed; + + public override string ToString() + { + return string.Format( + "created {0}, updated {1}, deleted {2}, failed {3}, references {4}", + Created, Updated, Deleted, Failed, References.Count); + } + } + + /// A batch of pending creates, built by a fixture one record at a time. + /// + /// PendingCreate is a private nested struct, so this wrapper is the only way a test + /// can hand one to a flush method. Same for the update and upsert batches below. + /// + public class TestCreateBatch + { + private readonly List Items = new List(); + + /// Adds a record to the batch, numbering positions from 1 as the import does. + public TestCreateBatch Add(Entity entity, Guid oldId = default(Guid), string identifier = null) + { + Items.Add(new PendingCreate + { + Entity = entity, + OldId = oldId == Guid.Empty ? entity.Id : oldId, + Position = Items.Count + 1, + Identifier = identifier ?? entity.LogicalName + " " + (Items.Count + 1) + }); + return this; + } + + /// How many records are queued. + public int Count => Items.Count; + + /// + /// The entities queued, in order. Read this before flushing: every + /// dispatcher clears its batch once it has flushed, so afterwards this is empty. + /// The list is a snapshot but the entities in it are the live objects, which is + /// what makes it useful for asserting ids written back by a bulk message. + /// + public IReadOnlyList Entities => Items.Select(i => i.Entity).ToList(); + /// + /// The flush calls live here rather than on the outer class: a containing type cannot + /// reach a nested type's private members, but a nested type can reach the outer's. + /// Items has to stay private, because PendingCreate is private (CS0052). + /// + internal BatchOutcome FlushDispatcher(Shuffler owner) + { + var outcome = new BatchOutcome(); + owner.FlushPendingCreates(owner.container, Items, ref outcome.Created, ref outcome.Failed, outcome.References); + return outcome; + } + + internal BatchOutcome FlushExecuteMultiple(Shuffler owner) + { + var outcome = new BatchOutcome(); + owner.FlushCreatesWithExecuteMultiple(owner.container, Items, ref outcome.Created, ref outcome.Failed, outcome.References); + return outcome; + } + + internal BatchOutcome FlushIndividually(Shuffler owner) + { + var outcome = new BatchOutcome(); + owner.FlushCreatesIndividually(owner.container, Items, ref outcome.Created, ref outcome.Failed, outcome.References); + return outcome; + } + + internal bool References(Entity entity) + { + return ReferencesPendingCreate(entity, Items); + } + } + + /// A batch of pending updates. + public class TestUpdateBatch + { + private readonly List Items = new List(); + + public TestUpdateBatch Add(Entity entity, string identifier = null) + { + Items.Add(new PendingUpdate + { + Entity = entity, + Position = Items.Count + 1, + Identifier = identifier ?? entity.LogicalName + " " + (Items.Count + 1) + }); + return this; + } + + public int Count => Items.Count; + + public IReadOnlyList Entities => Items.Select(i => i.Entity).ToList(); + internal BatchOutcome FlushDispatcher(Shuffler owner) + { + var outcome = new BatchOutcome(); + owner.FlushPendingUpdates(owner.container, Items, ref outcome.Updated, ref outcome.Failed, outcome.References); + return outcome; + } + + internal BatchOutcome FlushExecuteMultiple(Shuffler owner) + { + var outcome = new BatchOutcome(); + owner.FlushUpdatesWithExecuteMultiple(owner.container, Items, ref outcome.Updated, ref outcome.Failed, outcome.References); + return outcome; + } + + internal BatchOutcome FlushIndividually(Shuffler owner) + { + var outcome = new BatchOutcome(); + owner.FlushUpdatesIndividually(owner.container, Items, ref outcome.Updated, ref outcome.Failed, outcome.References); + return outcome; + } + } + + /// A batch of pending upserts. + public class TestUpsertBatch + { + private readonly List Items = new List(); + + public TestUpsertBatch Add(Entity entity, Guid oldId = default(Guid), string identifier = null) + { + Items.Add(new PendingUpsert + { + Entity = entity, + OldId = oldId == Guid.Empty ? entity.Id : oldId, + Position = Items.Count + 1, + Identifier = identifier ?? entity.LogicalName + " " + (Items.Count + 1) + }); + return this; + } + + public int Count => Items.Count; + + public IReadOnlyList Entities => Items.Select(i => i.Entity).ToList(); + internal BatchOutcome FlushDispatcher(Shuffler owner) + { + var outcome = new BatchOutcome(); + owner.FlushPendingUpserts(owner.container, Items, ref outcome.Created, ref outcome.Updated, ref outcome.Failed, outcome.References); + return outcome; + } + + internal BatchOutcome FlushAsCreateUpdate(Shuffler owner) + { + var outcome = new BatchOutcome(); + owner.FlushUpsertsAsCreateUpdate(owner.container, Items, ref outcome.Created, ref outcome.Updated, ref outcome.Failed, outcome.References); + return outcome; + } + } + + /// Runs the create dispatcher over . + public BatchOutcome TestFlushPendingCreates(TestCreateBatch batch) + { + return batch.FlushDispatcher(this); + } + + /// Runs the update dispatcher over . + public BatchOutcome TestFlushPendingUpdates(TestUpdateBatch batch) + { + return batch.FlushDispatcher(this); + } + + /// Runs the upsert dispatcher over . + public BatchOutcome TestFlushPendingUpserts(TestUpsertBatch batch) + { + return batch.FlushDispatcher(this); + } + + /// Calls the lowest upsert rung directly: create, then update on a duplicate. + public BatchOutcome TestFlushUpsertsAsCreateUpdate(TestUpsertBatch batch) + { + return batch.FlushAsCreateUpdate(this); + } + + /// Runs the delete dispatcher over . + public BatchOutcome TestFlushPendingDeletes(List batch) + { + var outcome = new BatchOutcome(); + FlushPendingDeletes(container, batch, ref outcome.Deleted, ref outcome.Failed); + return outcome; + } + + /// Calls the ExecuteMultiple rung for creates directly, skipping the dispatcher. + /// + /// The dispatcher probes for CreateMultiple first, so a fixture that wants to test + /// response-to-request pairing on its own would otherwise have to script the probe + /// as well. Going straight at the rung keeps those tests about one thing. + /// + public BatchOutcome TestFlushCreatesWithExecuteMultiple(TestCreateBatch batch) + { + return batch.FlushExecuteMultiple(this); + } + + /// Calls the ExecuteMultiple rung for updates directly. + public BatchOutcome TestFlushUpdatesWithExecuteMultiple(TestUpdateBatch batch) + { + return batch.FlushExecuteMultiple(this); + } + + /// Calls the individual-create rung directly. + public BatchOutcome TestFlushCreatesIndividually(TestCreateBatch batch) + { + return batch.FlushIndividually(this); + } + + /// Calls the individual-update rung directly. + public BatchOutcome TestFlushUpdatesIndividually(TestUpdateBatch batch) + { + return batch.FlushIndividually(this); + } + + /// Asks whether CreateMultiple is supported, driving the sdkmessagefilter probe. + public bool TestIsCreateMultipleSupported(string entityLogicalName) => + IsCreateMultipleSupported(container, entityLogicalName); + + /// Asks whether UpdateMultiple is supported. + public bool TestIsUpdateMultipleSupported(string entityLogicalName) => + IsUpdateMultipleSupported(container, entityLogicalName); + + /// Asks whether UpsertMultiple is supported. + public bool TestIsUpsertMultipleSupported(string entityLogicalName) => + IsUpsertMultipleSupported(container, entityLogicalName); + + /// Asks whether the single Upsert message is supported. + public bool TestIsUpsertSupported(string entityLogicalName) => + IsUpsertSupported(container, entityLogicalName); + + /// Whether a record can go in a batch at all. + public static bool TestIsBatchable(Entity entity) => IsBatchable(entity); + + /// Whether a record points at something still waiting in the create batch. + public static bool TestReferencesPendingCreate(Entity entity, TestCreateBatch pending) => + pending.References(entity); + + /// Rewrites every lookup on a record through the guid map, in place. + public void TestReplaceGuids(Entity entity, bool includeId = false) => + ReplaceGuids(container, entity, includeId); + + /// Adds a pair to the guid map, subject to the same conditions the import applies. + public void TestMapGuid(Guid oldId, Guid newId) => MapGuid(oldId, newId); + + /// Maps a created id and fills any deferred change waiting on that record. + public void TestRecordCreatedId(Guid oldId, Guid newId) => RecordCreatedId(oldId, newId); + + /// Records a batch error and reports whether the import should stop. + public bool TestStopOnBatchError(int position, string identifier) => + StopOnBatchError(position, identifier); + + /// Strips state and owner off a record, deferring them to the second pass. + public void TestStripAndDeferStateOwner(Entity entity, int position = 1, string identifier = null) + { + StripAndDeferStateOwner(entity, deferredStates, deferredOwners, position, identifier ?? entity.LogicalName); + } + + /// Queues a deferred state change without going through the strip pass. + public void TestDeferState(string entityLogicalName, Guid originalId, Guid actualId, int stateCode, int statusCode, int position = 1, string identifier = null) + { + deferredStates.Add(new DeferredStateChange + { + EntityLogicalName = entityLogicalName, + OriginalId = originalId, + ActualId = actualId, + StateCode = new OptionSetValue(stateCode), + StatusCode = new OptionSetValue(statusCode), + Position = position, + Identifier = identifier ?? entityLogicalName + }); + } + + /// Queues a deferred owner change without going through the strip pass. + public void TestDeferOwner(string entityLogicalName, Guid originalId, Guid actualId, EntityReference owner, int position = 1, string identifier = null) + { + deferredOwners.Add(new DeferredOwnerChange + { + EntityLogicalName = entityLogicalName, + OriginalId = originalId, + ActualId = actualId, + Owner = owner, + Position = position, + Identifier = identifier ?? entityLogicalName + }); + } + + /// How many state changes are waiting for the second pass. + public int TestDeferredStateCount => deferredStates.Count; + + /// How many owner changes are waiting for the second pass. + public int TestDeferredOwnerCount => deferredOwners.Count; + + /// The state codes queued for , as state and status. + public Tuple TestDeferredStateCodes(Guid originalId) + { + var match = deferredStates.Where(s => s.OriginalId == originalId).ToList(); + if (match.Count == 0) + { + return null; + } + return Tuple.Create(match[0].StateCode.Value, match[0].StatusCode.Value); + } + + /// + /// The id the deferred pass will actually write to for . + /// + /// + /// Empty means the record was never written, which is what the deferred pass uses to + /// decide to drop the change rather than update a record that does not exist. + /// + public Guid? TestDeferredActualId(Guid originalId) + { + var match = deferredStates.Where(s => s.OriginalId == originalId).ToList(); + return match.Count == 0 ? (Guid?)null : match[0].ActualId; + } + + /// The id the deferred owner pass will write to for . + public Guid? TestDeferredOwnerActualId(Guid originalId) + { + var match = deferredOwners.Where(o => o.OriginalId == originalId).ToList(); + return match.Count == 0 ? (Guid?)null : match[0].ActualId; + } + + /// Fills in real ids on the deferred queues after a record was written. + public void TestUpdateDeferredActualIds(Guid originalId, Guid actualId) => + UpdateDeferredActualIds(originalId, actualId); + + /// Runs the deferred state pass. + public void TestFlushDeferredStateChanges() => + FlushDeferredStateChanges(container, deferredStates); + + /// What a whole block import did, by counter. + public class BlockOutcome + { + public int Created; + public int Updated; + public int Skipped; + public int Deleted; + public int Failed; + public EntityReferenceCollection References; + + /// Every record the block accounted for, however it accounted for it. + public int Accounted + { + get { return Created + Updated + Skipped + Deleted + Failed; } + } + + public override string ToString() + { + return string.Format( + "created {0}, updated {1}, skipped {2}, deleted {3}, failed {4}", + Created, Updated, Skipped, Deleted, Failed); + } + } + + /// + /// Runs one whole data block, which is the only way to reach the decisions that are + /// made before any flush happens - the upsert gate, batchability, match resolution and + /// the capability probes. + /// + public BlockOutcome TestImportDataBlock(Types.DataBlock block, EntityCollection entities) + { + var result = ImportDataBlock(container, block, entities); + return new BlockOutcome + { + Created = result.Item1, + Updated = result.Item2, + Skipped = result.Item3, + Deleted = result.Item4, + Failed = result.Item5, + References = result.Item6 + }; + } + + /// Runs the deferred owner pass. + public void TestFlushDeferredOwnerChanges() => + FlushDeferredOwnerChanges(container, deferredOwners); + + /// + /// Runs the export attribute filter over a collection of exported records. + /// + /// + /// Static, and takes its own container, because SelectAttributes is static on the + /// product side too - it needs no shuffler state, only metadata for the primary id. + /// + public static void TestSelectAttributes(IExecutionContainer container, EntityCollection entities, List attributes, List nullAttributes) => + SelectAttributes(container, entities, attributes, nullAttributes); + + /// Sends one line through the product formatting path. + public void TestSendLine(string msg, params object[] args) => + SendLine(container, msg, args); + + /// Sends a bare newline - the case the length guard in SendText drops. + public void TestSendBlankLine() => SendLine(container); + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Helpers/TestExecutionContainer.cs b/tests/Xrm.Shuffle.Core.Tests/Helpers/TestExecutionContainer.cs new file mode 100644 index 0000000..61f5946 --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Helpers/TestExecutionContainer.cs @@ -0,0 +1,39 @@ +namespace Cinteros.Crm.Utils.Shuffle.Tests.Helpers +{ + using System.Dynamic; + using Microsoft.Xrm.Sdk; + using global::Xrm.Utils.Core.Common.Interfaces; + + /// + /// A minimal . + /// + /// + /// The interface is three read-only properties, so this is a stub rather than a mock - + /// there is no behaviour here worth a mocking framework. + /// + public class TestExecutionContainer : IExecutionContainer + { + public TestExecutionContainer(IOrganizationService service) + : this(service, new RecordingLogger()) + { + } + + public TestExecutionContainer(IOrganizationService service, RecordingLogger logger) + { + Service = service; + Recorder = logger; + // The product reads and writes container.Values as a dynamic bag; ExpandoObject + // is what CintContainer uses too. + Values = new ExpandoObject(); + } + + public dynamic Values { get; } + + public ILoggable Logger => Recorder; + + public IOrganizationService Service { get; } + + /// The same object as , typed so tests can read it. + public RecordingLogger Recorder { get; } + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Layer1/BatchResponsePairingTests.cs b/tests/Xrm.Shuffle.Core.Tests/Layer1/BatchResponsePairingTests.cs new file mode 100644 index 0000000..bd6218e --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Layer1/BatchResponsePairingTests.cs @@ -0,0 +1,177 @@ +namespace Cinteros.Crm.Utils.Shuffle.Tests.Layer1 +{ + using System; + using System.Linq; + using Cinteros.Crm.Utils.Shuffle; + using Cinteros.Crm.Utils.Shuffle.Tests.Helpers; + using Microsoft.Xrm.Sdk; + using NUnit.Framework; + + /// + /// How the ExecuteMultiple rungs pair response items back to the requests that produced + /// them. This is where the counters come from, so a pairing bug shows up as a run that + /// reports more work than it did. + /// + /// + /// These go straight at the ExecuteMultiple rung rather than through the dispatcher, so + /// that no capability probe has to be scripted and each test is about one thing. The + /// dispatcher's own routing is covered separately. + /// + [TestFixture] + public class BatchResponsePairingTests : ShuffleTestBase + { + private const string ExecuteMultiple = "ExecuteMultiple"; + + private static Shuffler.TestCreateBatch Creates(int count) + { + var batch = new Shuffler.TestCreateBatch(); + for (var i = 1; i <= count; i++) + { + batch.Add(Record("account", Id(i), "Account " + i), Id(i), "account Acme " + i); + } + return batch; + } + + [Test] + public void One_fault_in_a_batch_of_twenty_is_one_failure_and_nineteen_successes() + { + // The regression behind 4f9233d: the fault used to end the accounting, so the + // records after it were never counted at all and the totals did not add up. + var responses = new ExecuteMultipleResponseBuilder(); + for (var i = 0; i < 20; i++) + { + if (i == 11) + { + responses.Fault("Nope"); + } + else + { + responses.CreatedAt(Id(100 + i)); + } + } + Service.OnMessage(ExecuteMultiple, responses.Build()); + + var outcome = NewShuffler().TestFlushCreatesWithExecuteMultiple(Creates(20)); + + Assert.That(outcome.Created, Is.EqualTo(19), outcome.ToString()); + Assert.That(outcome.Failed, Is.EqualTo(1), outcome.ToString()); + Assert.That(outcome.Accounted, Is.EqualTo(20), "every request must be accounted for exactly once"); + Recorder.AssertSent("012 Create Failed: account Acme 12 Nope"); + Recorder.AssertSent("013 Created: account Acme 13"); + } + + [Test] + public void Responses_are_matched_by_request_index_not_by_position() + { + // The platform is free to return items in any order; the loop looks them up by + // RequestIndex for exactly that reason. + Service.OnMessage(ExecuteMultiple, new ExecuteMultipleResponseBuilder() + .SuccessAt(2, CreateResponseFor(Id(30))) + .FaultAt(0, "First one failed") + .SuccessAt(1, CreateResponseFor(Id(20))) + .Build()); + + var batch = Creates(3); + var outcome = NewShuffler().TestFlushCreatesWithExecuteMultiple(batch); + + Assert.That(outcome.Created, Is.EqualTo(2), outcome.ToString()); + Assert.That(outcome.Failed, Is.EqualTo(1), outcome.ToString()); + Recorder.AssertSent("001 Create Failed: account Acme 1 First one failed"); + Assert.That(batch.Entities[1].Id, Is.EqualTo(Id(20)), "the id must come from the matching response"); + Assert.That(batch.Entities[2].Id, Is.EqualTo(Id(30))); + } + + [Test] + public void Requests_the_platform_never_answered_count_as_failures_not_successes() + { + // ContinueOnError=false stops the platform at the first fault, so the collection + // is shorter than the request list. Nothing after the hole was executed. + Service.OnMessage(ExecuteMultiple, new ExecuteMultipleResponseBuilder() + .CreatedAt(Id(10)) + .Fault("Bad row") + .Omit() + .Omit() + .Build()); + + var outcome = NewShuffler().TestFlushCreatesWithExecuteMultiple(Creates(4)); + + Assert.That(outcome.Created, Is.EqualTo(1), outcome.ToString()); + Assert.That(outcome.Failed, Is.EqualTo(3), outcome.ToString()); + Assert.That(outcome.Accounted, Is.EqualTo(4)); + Recorder.AssertSent("003 Create Not Executed: account Acme 3"); + Recorder.AssertSent("004 Create Not Executed: account Acme 4"); + } + + [Test] + public void A_created_record_takes_the_id_the_platform_returned() + { + Service.OnMessage(ExecuteMultiple, new ExecuteMultipleResponseBuilder() + .CreatedAt(Id(901)) + .CreatedAt(Id(902)) + .Build()); + + var batch = Creates(2); + var outcome = NewShuffler().TestFlushCreatesWithExecuteMultiple(batch); + + Assert.That(batch.Entities[0].Id, Is.EqualTo(Id(901))); + Assert.That(batch.Entities[1].Id, Is.EqualTo(Id(902))); + Assert.That(outcome.References.Select(r => r.Id).ToArray(), Is.EqualTo(new[] { Id(901), Id(902) })); + } + + [Test] + public void A_batch_create_that_throws_falls_back_to_one_create_per_record() + { + Service.OnMessage(ExecuteMultiple, request => + { + throw ExecuteMultipleResponseBuilder.Faulted("Batch too large"); + }); + Service.OnCreate(entity => Id(500)); + + var outcome = NewShuffler().TestFlushCreatesWithExecuteMultiple(Creates(3)); + + Assert.That(outcome.Created, Is.EqualTo(3), outcome.ToString()); + Assert.That(Service.CountOf("Create"), Is.EqualTo(3), "one individual create per record"); + Recorder.AssertLogged("Falling back to sequential creates"); + } + + [Test] + public void A_batch_create_that_throws_under_StopOnError_does_not_fall_back() + { + Service.OnMessage(ExecuteMultiple, request => + { + throw ExecuteMultipleResponseBuilder.Faulted("Batch too large"); + }); + + var shuffler = NewShuffler(stopOnError: true); + var batch = Creates(3); + + Assert.Throws>( + () => shuffler.TestFlushCreatesWithExecuteMultiple(batch)); + Assert.That(Service.CountOf("Create"), Is.EqualTo(0), "StopOnError must not retry the records one by one"); + } + + [Test] + public void A_fault_under_StopOnError_aborts_the_batch_and_names_the_record() + { + Service.OnMessage(ExecuteMultiple, new ExecuteMultipleResponseBuilder() + .CreatedAt(Id(10)) + .Fault("Bad row") + .CreatedAt(Id(30)) + .Build()); + + var shuffler = NewShuffler(stopOnError: true); + + Assert.Throws(() => shuffler.TestFlushCreatesWithExecuteMultiple(Creates(3))); + Assert.That(shuffler.TestBatchFailureLabel, Is.EqualTo("002 account Acme 2")); + Recorder.AssertLogged("StopOnError: aborting, 1 record(s) in this batch were not executed"); + Recorder.AssertNeverSent("003 Created: account Acme 3"); + } + + private static OrganizationResponse CreateResponseFor(Guid id) + { + var response = new Microsoft.Xrm.Sdk.Messages.CreateResponse(); + response.Results["id"] = id; + return response; + } + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Layer1/BulkMessageFallbackTests.cs b/tests/Xrm.Shuffle.Core.Tests/Layer1/BulkMessageFallbackTests.cs new file mode 100644 index 0000000..b3f5b4e --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Layer1/BulkMessageFallbackTests.cs @@ -0,0 +1,297 @@ +namespace Cinteros.Crm.Utils.Shuffle.Tests.Layer1 +{ + using System; + using System.Linq; + using System.ServiceModel; + using Cinteros.Crm.Utils.Shuffle; + using Cinteros.Crm.Utils.Shuffle.Tests.Helpers; + using Microsoft.Xrm.Sdk; + using Microsoft.Xrm.Sdk.Messages; + using NUnit.Framework; + + /// + /// What happens when a bulk message is asked for and does not answer. There are two very + /// different cases and the product must not confuse them: the message does not exist on + /// this org, which is permanent and should be remembered; or the message exists and the + /// batch failed, which says nothing about the next batch. + /// + /// + /// The second case also has to undo itself. CreateMultiple, UpdateMultiple and + /// UpsertMultiple are each a single transaction, so a fault rolled the whole batch back + /// and nothing was written - the rows have to be re-run one at a time, both to get the + /// records ahead of the bad one committed and to name the one that actually failed. + /// + [TestFixture] + public class BulkMessageFallbackTests : ShuffleTestBase + { + private const string CreateMultiple = "CreateMultiple"; + private const string UpdateMultiple = "UpdateMultiple"; + private const string UpsertMultiple = "UpsertMultiple"; + private const string ExecuteMultiple = "ExecuteMultiple"; + + private static Shuffler.TestCreateBatch Creates(int count) + { + var batch = new Shuffler.TestCreateBatch(); + for (var i = 1; i <= count; i++) + { + batch.Add(Record("account", Id(i), "Account " + i), Id(i), "account Acme " + i); + } + return batch; + } + + private static Shuffler.TestUpdateBatch Updates(int count) + { + var batch = new Shuffler.TestUpdateBatch(); + for (var i = 1; i <= count; i++) + { + batch.Add(Record("account", Id(i), "Account " + i), "account Acme " + i); + } + return batch; + } + + private static Shuffler.TestUpsertBatch Upserts(int count) + { + var batch = new Shuffler.TestUpsertBatch(); + for (var i = 1; i <= count; i++) + { + batch.Add(Record("account", Id(i), "Account " + i), Id(i), "account Acme " + i); + } + return batch; + } + + private static ExecuteMultipleResponse TwoCreates() + { + return new ExecuteMultipleResponseBuilder().CreatedAt(Id(101)).CreatedAt(Id(102)).Build(); + } + + [Test] + public void A_not_implemented_CreateMultiple_falls_back_to_ExecuteMultiple() + { + // 0x80040265 is the on-premises answer: the message is not on this org at all. + Service.SupportsBulkMessage("account", CreateMultiple); + Service.ThrowOnce(CreateMultiple, ExecuteMultipleResponseBuilder.MessageNotImplemented(CreateMultiple)); + Service.OnMessage(ExecuteMultiple, TwoCreates()); + + var outcome = NewShuffler().TestFlushPendingCreates(Creates(2)); + + Assert.That(outcome.Created, Is.EqualTo(2), outcome.ToString()); + Assert.That(outcome.Failed, Is.EqualTo(0), outcome.ToString()); + Assert.That(Service.CountOf(ExecuteMultiple), Is.EqualTo(1), DumpAll()); + Recorder.AssertLogged("CreateMultiple not implemented, marking as unsupported and falling back"); + } + + [Test] + public void A_NotSupportedException_is_read_as_not_implemented_too() + { + // Some channels surface an absent message this way rather than as a fault. + Service.SupportsBulkMessage("account", CreateMultiple); + Service.ThrowOnce(CreateMultiple, new NotSupportedException("no such message")); + Service.OnMessage(ExecuteMultiple, TwoCreates()); + + NewShuffler().TestFlushPendingCreates(Creates(2)); + + Assert.That(Service.CountOf(ExecuteMultiple), Is.EqualTo(1), DumpAll()); + Recorder.AssertLogged("CreateMultiple not implemented"); + } + + [Test] + public void A_not_implemented_fault_wrapped_in_another_exception_is_still_found() + { + // The check walks InnerException, because the proxy layer wraps. + Service.SupportsBulkMessage("account", CreateMultiple); + Service.ThrowOnce(CreateMultiple, new InvalidOperationException( + "wrapped", ExecuteMultipleResponseBuilder.MessageNotImplemented(CreateMultiple))); + Service.OnMessage(ExecuteMultiple, TwoCreates()); + + NewShuffler().TestFlushPendingCreates(Creates(2)); + + Assert.That(Service.CountOf(ExecuteMultiple), Is.EqualTo(1), DumpAll()); + Recorder.AssertLogged("CreateMultiple not implemented"); + } + + [Test] + public void An_org_that_answered_not_implemented_once_is_not_asked_again() + { + // Without the cache every batch for the rest of the run pays a doomed round trip. + Service.SupportsBulkMessage("account", CreateMultiple); + Service.ThrowOnce(CreateMultiple, ExecuteMultipleResponseBuilder.MessageNotImplemented(CreateMultiple)); + Service.OnMessage(ExecuteMultiple, TwoCreates()); + + var shuffler = NewShuffler(); + shuffler.TestFlushPendingCreates(Creates(2)); + shuffler.TestFlushPendingCreates(Creates(2)); + + Assert.That(Service.CountOf(CreateMultiple), Is.EqualTo(1), DumpAll()); + Assert.That(Service.CountOf(ExecuteMultiple), Is.EqualTo(2), DumpAll()); + } + + [Test] + public void An_ordinary_CreateMultiple_fault_re_runs_the_rows_one_at_a_time() + { + // The batch was one transaction, so nothing was written. Going to ExecuteMultiple + // instead would be wrong twice over: the message does exist, and the per-record + // path is the only one that can name the row that faulted. + Service.SupportsBulkMessage("account", CreateMultiple); + Service.ThrowOnce(CreateMultiple, ExecuteMultipleResponseBuilder.Faulted("bad row somewhere")); + var ids = new System.Collections.Generic.Queue(new[] { Id(901), Id(902) }); + Service.OnCreate(e => ids.Dequeue()); + + var outcome = NewShuffler().TestFlushPendingCreates(Creates(2)); + + Assert.That(outcome.Created, Is.EqualTo(2), outcome.ToString()); + Assert.That(Service.CountOf(ExecuteMultiple), Is.EqualTo(0), DumpAll()); + Assert.That(Service.Created.Count, Is.EqualTo(2), DumpAll()); + Recorder.AssertLogged("CreateMultiple batch failed, falling back to individual creates"); + Recorder.AssertNeverLogged("marking as unsupported"); + } + + [Test] + public void The_row_that_faulted_the_batch_is_named_by_the_re_run() + { + Service.SupportsBulkMessage("account", CreateMultiple); + Service.ThrowOnce(CreateMultiple, ExecuteMultipleResponseBuilder.Faulted("bad row somewhere")); + Service.OnCreate(Refuse(Id(2), "duplicate name")); + + var outcome = NewShuffler().TestFlushPendingCreates(Creates(3)); + + Assert.That(outcome.Created, Is.EqualTo(2), outcome.ToString()); + Assert.That(outcome.Failed, Is.EqualTo(1), outcome.ToString()); + Recorder.AssertSent("002 Create Failed: account Acme 2 duplicate name"); + Recorder.AssertSent("003 Created: account Acme 3"); + } + + [Test] + public void StopOnError_does_not_stop_the_re_run_from_starting() + { + // Deliberate: with the batch rolled back, refusing to re-run would abort the + // import without ever saying which record was at fault. The per-record path + // honours StopOnError itself, once it has named the row. + Service.SupportsBulkMessage("account", CreateMultiple); + Service.ThrowOnce(CreateMultiple, ExecuteMultipleResponseBuilder.Faulted("bad row somewhere")); + Service.OnCreate(Refuse(Id(2), "duplicate name")); + + var shuffler = NewShuffler(stopOnError: true); + + Assert.Throws(() => shuffler.TestFlushPendingCreates(Creates(3))); + Recorder.AssertSent("002 Create Failed: account Acme 2 duplicate name"); + Recorder.AssertNeverSent("003 Created: account Acme 3"); + Assert.That(shuffler.TestBatchFailureLabel, Is.EqualTo("002 account Acme 2")); + } + + [Test] + public void A_not_implemented_UpdateMultiple_falls_back_to_ExecuteMultiple() + { + Service.SupportsBulkMessage("account", UpdateMultiple); + Service.ThrowOnce(UpdateMultiple, ExecuteMultipleResponseBuilder.MessageNotImplemented(UpdateMultiple)); + Service.OnMessage(ExecuteMultiple, new ExecuteMultipleResponseBuilder().Succeeded().Succeeded().Build()); + + var outcome = NewShuffler().TestFlushPendingUpdates(Updates(2)); + + Assert.That(outcome.Updated, Is.EqualTo(2), outcome.ToString()); + Assert.That(Service.CountOf(ExecuteMultiple), Is.EqualTo(1), DumpAll()); + Recorder.AssertLogged("UpdateMultiple not implemented"); + } + + [Test] + public void An_ordinary_UpdateMultiple_fault_re_runs_the_rows_one_at_a_time() + { + Service.SupportsBulkMessage("account", UpdateMultiple); + Service.ThrowOnce(UpdateMultiple, ExecuteMultipleResponseBuilder.Faulted("bad row somewhere")); + + var outcome = NewShuffler().TestFlushPendingUpdates(Updates(2)); + + Assert.That(outcome.Updated, Is.EqualTo(2), outcome.ToString()); + Assert.That(Service.CountOf(ExecuteMultiple), Is.EqualTo(0), DumpAll()); + Assert.That(Service.Updated.Count, Is.EqualTo(2), DumpAll()); + Recorder.AssertLogged("UpdateMultiple batch failed, falling back to individual updates"); + } + + [Test] + public void A_not_implemented_UpsertMultiple_drops_to_ExecuteMultiple_carrying_Upsert_requests() + { + // Upsert has one more rung than create and update: a batch of single Upserts + // before it gives up and does Create/Update per record. + Service.SupportsBulkMessage("account", UpsertMultiple, "Upsert"); + Service.ThrowOnce(UpsertMultiple, ExecuteMultipleResponseBuilder.MessageNotImplemented(UpsertMultiple)); + Service.OnMessage(ExecuteMultiple, new ExecuteMultipleResponseBuilder() + .Success(Upserted(true, Id(901))) + .Success(Upserted(false, Id(2))) + .Build()); + + var outcome = NewShuffler().TestFlushPendingUpserts(Upserts(2)); + + Assert.That(outcome.Created, Is.EqualTo(1), outcome.ToString()); + Assert.That(outcome.Updated, Is.EqualTo(1), outcome.ToString()); + var inner = (ExecuteMultipleRequest)Service.RequestsNamed(ExecuteMultiple).Single(); + Assert.That(inner.Requests.All(r => r is UpsertRequest), Is.True, DumpAll()); + Recorder.AssertLogged("UpsertMultiple not implemented"); + } + + [Test] + public void An_ordinary_UpsertMultiple_fault_retries_as_a_batch_of_single_upserts() + { + // Unlike create and update, this one does not drop to per-record work: Upsert is + // idempotent, so the cheaper rung is tried first and Create/Update is left as the + // last resort. + Service.SupportsBulkMessage("account", UpsertMultiple, "Upsert"); + Service.ThrowOnce(UpsertMultiple, ExecuteMultipleResponseBuilder.Faulted("bad row somewhere")); + Service.OnMessage(ExecuteMultiple, new ExecuteMultipleResponseBuilder() + .Success(Upserted(true, Id(901))) + .Success(Upserted(true, Id(902))) + .Build()); + + var outcome = NewShuffler().TestFlushPendingUpserts(Upserts(2)); + + Assert.That(outcome.Created, Is.EqualTo(2), outcome.ToString()); + Assert.That(Service.CountOf(ExecuteMultiple), Is.EqualTo(1), DumpAll()); + Assert.That(Service.Created.Count, Is.EqualTo(0), "Create/Update is the rung below this one"); + Recorder.AssertLogged("UpsertMultiple batch failed, falling back to ExecuteMultiple with Upsert"); + } + + [Test] + public void Known_defect_a_late_not_implemented_upsert_fault_counts_the_rows_ahead_of_it_twice() + { + // TryFlushUpsertsWithExecuteMultiple can discover "Upsert not implemented" from a + // response item partway down the batch and return false - but the rows before it + // have already been counted, and the caller then re-runs the whole batch through + // Create/Update. Two records go in, three are reported. + // + // Asserted as it behaves today, deliberately. A fix should either roll the + // counters back or finish the batch, and either way this test should then fail + // and be rewritten rather than quietly keep passing. + Service.SupportsBulkMessage("account", "Upsert"); + Service.OnMessage(ExecuteMultiple, new ExecuteMultipleResponseBuilder() + .Success(Upserted(true, Id(901))) + .Fault("not implemented here", unchecked((int)0x80040265)) + .Build()); + Service.OnCreate(e => Id(902)); + + var outcome = NewShuffler().TestFlushPendingUpserts(Upserts(2)); + + Assert.That(outcome.Created, Is.EqualTo(3), + "known defect: the first row is counted by both the upsert rung and the re-run"); + Assert.That(Service.Created.Count, Is.EqualTo(2), "only two records were actually written"); + } + + /// A create handler that refuses one record and accepts the rest. + private static Func Refuse(Guid id, string message) + { + return entity => + { + if (entity.Id == id) + { + throw new InvalidOperationException(message); + } + return Id(901); + }; + } + + private static UpsertResponse Upserted(bool recordCreated, Guid id) + { + var response = new UpsertResponse(); + response.Results["RecordCreated"] = recordCreated; + response.Results["Target"] = new EntityReference("account", id); + return response; + } + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Layer1/BulkMessageRoutingTests.cs b/tests/Xrm.Shuffle.Core.Tests/Layer1/BulkMessageRoutingTests.cs new file mode 100644 index 0000000..67cc0f5 --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Layer1/BulkMessageRoutingTests.cs @@ -0,0 +1,265 @@ +namespace Cinteros.Crm.Utils.Shuffle.Tests.Layer1 +{ + using System; + using System.Linq; + using Cinteros.Crm.Utils.Shuffle; + using Cinteros.Crm.Utils.Shuffle.Tests.Helpers; + using Microsoft.Xrm.Sdk; + using Microsoft.Xrm.Sdk.Messages; + using NUnit.Framework; + + /// + /// Which message a batch actually goes out as. The dispatchers pick between the bulk + /// message, ExecuteMultiple and one request per record, and the choice is invisible from + /// the counters - a batch that quietly took the slow path reports exactly the same totals + /// as one that took the fast one. So every test here asserts the routing it expected. + /// + [TestFixture] + public class BulkMessageRoutingTests : ShuffleTestBase + { + private const string CreateMultiple = "CreateMultiple"; + private const string UpdateMultiple = "UpdateMultiple"; + private const string UpsertMultiple = "UpsertMultiple"; + private const string ExecuteMultiple = "ExecuteMultiple"; + + private static Shuffler.TestCreateBatch Creates(int count) + { + var batch = new Shuffler.TestCreateBatch(); + for (var i = 1; i <= count; i++) + { + batch.Add(Record("account", Id(i), "Account " + i), Id(i), "account Acme " + i); + } + return batch; + } + + private static Shuffler.TestUpdateBatch Updates(int count) + { + var batch = new Shuffler.TestUpdateBatch(); + for (var i = 1; i <= count; i++) + { + batch.Add(Record("account", Id(i), "Account " + i), "account Acme " + i); + } + return batch; + } + + private static Shuffler.TestUpsertBatch Upserts(int count) + { + var batch = new Shuffler.TestUpsertBatch(); + for (var i = 1; i <= count; i++) + { + batch.Add(Record("account", Id(i), "Account " + i), Id(i), "account Acme " + i); + } + return batch; + } + + private static OrganizationResponse CreateMultipleReturning(params Guid[] ids) + { + var response = new OrganizationResponse(); + response.Results["Ids"] = ids; + return response; + } + + [Test] + public void A_supported_entity_creates_through_CreateMultiple_and_never_reaches_ExecuteMultiple() + { + Service.SupportsBulkMessage("account", CreateMultiple); + Service.OnMessage(CreateMultiple, CreateMultipleReturning(Id(101), Id(102), Id(103))); + + var outcome = NewShuffler().TestFlushPendingCreates(Creates(3)); + + Assert.That(outcome.Created, Is.EqualTo(3), outcome.ToString()); + Assert.That(Service.CountOf(CreateMultiple), Is.EqualTo(1), DumpAll()); + Assert.That(Service.CountOf(ExecuteMultiple), Is.EqualTo(0), DumpAll()); + Assert.That(Service.Created.Count, Is.EqualTo(0), "nothing should have gone out one at a time"); + } + + [Test] + public void One_CreateMultiple_carries_every_record_in_the_batch() + { + Service.SupportsBulkMessage("account", CreateMultiple); + Service.OnMessage(CreateMultiple, CreateMultipleReturning(Id(101), Id(102), Id(103))); + + NewShuffler().TestFlushPendingCreates(Creates(3)); + + var targets = (EntityCollection)Service.RequestsNamed(CreateMultiple).Single()["Targets"]; + Assert.That(targets.EntityName, Is.EqualTo("account")); + Assert.That(targets.Entities.Count, Is.EqualTo(3)); + } + + [Test] + public void The_ids_CreateMultiple_returns_are_written_back_onto_the_records() + { + // The platform allocates the ids, so the batch has to take them back or every + // later block that points at these records maps to the wrong row. + Service.SupportsBulkMessage("account", CreateMultiple); + Service.OnMessage(CreateMultiple, CreateMultipleReturning(Id(901), Id(902))); + + var batch = Creates(2); + + // Held onto before the flush: the dispatcher clears the batch once it has flushed, + // so reading batch.Entities afterwards yields nothing. The records themselves are + // the same objects, and it is their Id the batch writes to. + var records = batch.Entities; + var outcome = NewShuffler().TestFlushPendingCreates(batch); + + Assert.That(records.Select(e => e.Id).ToArray(), Is.EqualTo(new[] { Id(901), Id(902) })); + Assert.That(outcome.References.Select(r => r.Id).ToArray(), Is.EqualTo(new[] { Id(901), Id(902) })); + } + + [Test] + public void An_entity_without_CreateMultiple_goes_straight_to_ExecuteMultiple() + { + Service.OnMessage(ExecuteMultiple, new ExecuteMultipleResponseBuilder() + .CreatedAt(Id(101)).CreatedAt(Id(102)).Build()); + + var outcome = NewShuffler().TestFlushPendingCreates(Creates(2)); + + Assert.That(outcome.Created, Is.EqualTo(2), outcome.ToString()); + Assert.That(Service.CountOf(CreateMultiple), Is.EqualTo(0), DumpAll()); + Assert.That(Service.CountOf(ExecuteMultiple), Is.EqualTo(1), DumpAll()); + } + + [Test] + public void A_supported_entity_updates_through_UpdateMultiple() + { + Service.SupportsBulkMessage("account", UpdateMultiple); + Service.OnMessage(UpdateMultiple, new OrganizationResponse()); + + var outcome = NewShuffler().TestFlushPendingUpdates(Updates(3)); + + Assert.That(outcome.Updated, Is.EqualTo(3), outcome.ToString()); + Assert.That(Service.CountOf(UpdateMultiple), Is.EqualTo(1), DumpAll()); + Assert.That(Service.CountOf(ExecuteMultiple), Is.EqualTo(0), DumpAll()); + } + + [Test] + public void Upsert_prefers_UpsertMultiple_over_the_single_Upsert_rung() + { + // Both are probed, UpsertMultiple first. An org that has gained both must not + // fall back to a batch of single Upserts. + Service.SupportsBulkMessage("account", UpsertMultiple, "Upsert"); + Service.OnMessage(UpsertMultiple, UpsertMultipleReturning( + UpsertResult(true, Id(901)), + UpsertResult(false, Id(2)))); + + var outcome = NewShuffler().TestFlushPendingUpserts(Upserts(2)); + + Assert.That(outcome.Created, Is.EqualTo(1), outcome.ToString()); + Assert.That(outcome.Updated, Is.EqualTo(1), outcome.ToString()); + Assert.That(Service.CountOf(UpsertMultiple), Is.EqualTo(1), DumpAll()); + Assert.That(Service.CountOf(ExecuteMultiple), Is.EqualTo(0), DumpAll()); + Recorder.AssertSent("001 Created (upsert): account Acme 1"); + Recorder.AssertSent("002 Updated (upsert): account Acme 2"); + } + + [Test] + public void An_upsert_result_the_platform_left_out_is_counted_as_an_update() + { + // Not knowing is not the same as failing: the row went in either way, and + // "Upserted" is the wording that says which of the two counters is a guess. + Service.SupportsBulkMessage("account", UpsertMultiple); + Service.OnMessage(UpsertMultiple, UpsertMultipleReturning(UpsertResult(true, Id(901)))); + + var outcome = NewShuffler().TestFlushPendingUpserts(Upserts(2)); + + Assert.That(outcome.Created, Is.EqualTo(1), outcome.ToString()); + Assert.That(outcome.Updated, Is.EqualTo(1), outcome.ToString()); + Assert.That(outcome.Failed, Is.EqualTo(0), outcome.ToString()); + Recorder.AssertSent("002 Upserted: account Acme 2"); + } + + [Test] + public void An_entity_with_neither_upsert_message_falls_all_the_way_to_create_and_update() + { + Service.OnCreate(e => Id(901)); + + var outcome = NewShuffler().TestFlushPendingUpserts(Upserts(2)); + + Assert.That(Service.CountOf(UpsertMultiple), Is.EqualTo(0), DumpAll()); + Assert.That(Service.CountOf(ExecuteMultiple), Is.EqualTo(0), DumpAll()); + Assert.That(outcome.Accounted, Is.EqualTo(2), outcome.ToString()); + } + + [Test] + public void The_capability_probe_runs_once_per_entity_and_message() + { + // It is a RetrieveMultiple against sdkmessagefilter, so an uncached probe would + // cost one extra round trip per batch for the whole run. + Service.SupportsBulkMessage("account", CreateMultiple); + Service.OnMessage(CreateMultiple, CreateMultipleReturning(Id(101), Id(102))); + + var shuffler = NewShuffler(); + shuffler.TestFlushPendingCreates(Creates(2)); + shuffler.TestFlushPendingCreates(Creates(2)); + + Assert.That(Service.CountOf(CreateMultiple), Is.EqualTo(2), DumpAll()); + Assert.That(Service.Probes.Count, Is.EqualTo(1), + "the second batch should have read the cached answer: " + + string.Join(", ", Service.Probes.Select(p => p.Item2 + "/" + p.Item1))); + } + + [Test] + public void A_probe_that_cannot_be_answered_is_treated_as_no_rather_than_failing_the_import() + { + // A locked-down org can refuse the sdkmessagefilter query outright. That is a + // reason to use the slow path, not a reason to abandon the import. + Service.FailTheCapabilityProbe(new InvalidOperationException("no privilege")); + Service.OnMessage(ExecuteMultiple, new ExecuteMultipleResponseBuilder() + .CreatedAt(Id(101)).CreatedAt(Id(102)).Build()); + + var outcome = NewShuffler().TestFlushPendingCreates(Creates(2)); + + Assert.That(outcome.Created, Is.EqualTo(2), outcome.ToString()); + Assert.That(Service.CountOf(ExecuteMultiple), Is.EqualTo(1), DumpAll()); + Recorder.AssertLogged("Failed to check CreateMultiple support for account"); + } + + [Test] + public void A_single_record_batch_never_probes_for_bulk_support() + { + // The Count == 1 shortcut is ahead of the probe, which is what makes BatchSize=1 + // cost nothing at all rather than one sdkmessagefilter query per record. + Service.OnCreate(e => Id(901)); + + var outcome = NewShuffler().TestFlushPendingCreates(Creates(1)); + + Assert.That(outcome.Created, Is.EqualTo(1), outcome.ToString()); + Assert.That(Service.Probes.Count, Is.EqualTo(0), DumpAll()); + Assert.That(Service.Created.Count, Is.EqualTo(1), DumpAll()); + } + + [Test] + public void Capability_query_uses_the_logical_name_not_the_entity_type_code() + { + // Landmine marker, not an endorsement. sdkmessagefilter.primaryobjecttypecode + // holds a numeric entity type code, but the product queries it with the logical + // name string - and the test double matches the product, so a fixture that says + // an entity supports CreateMultiple gets the answer it asked for. + // + // Against a real org that comparison is what makes the probe answer no for + // everything, which is why every bulk path also has a working fallback. If this + // is ever fixed to resolve the type code first, this test fails, and the fixtures + // that call SupportsBulkMessage need the same treatment. + NewShuffler().TestIsCreateMultipleSupported("account"); + + Assert.That(Service.Probes.Count, Is.EqualTo(1)); + Assert.That(Service.Probes[0].Item1, Is.EqualTo("account")); + Assert.That(Service.Probes[0].Item2, Is.EqualTo(CreateMultiple)); + } + + private static UpsertResponse UpsertResult(bool recordCreated, Guid id) + { + var response = new UpsertResponse(); + response.Results["RecordCreated"] = recordCreated; + response.Results["Target"] = new EntityReference("account", id); + return response; + } + + private static OrganizationResponse UpsertMultipleReturning(params UpsertResponse[] results) + { + var response = new OrganizationResponse(); + response.Results["Results"] = results; + return response; + } + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Layer1/UpdateAndDeletePairingTests.cs b/tests/Xrm.Shuffle.Core.Tests/Layer1/UpdateAndDeletePairingTests.cs new file mode 100644 index 0000000..285ed6c --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Layer1/UpdateAndDeletePairingTests.cs @@ -0,0 +1,162 @@ +namespace Cinteros.Crm.Utils.Shuffle.Tests.Layer1 +{ + using System; + using System.Collections.Generic; + using Cinteros.Crm.Utils.Shuffle; + using Cinteros.Crm.Utils.Shuffle.Tests.Helpers; + using Microsoft.Xrm.Sdk; + using NUnit.Framework; + + /// + /// The same response pairing for the update and delete rungs. They are separate loops in + /// the product with their own log wording, so they need their own tests rather than a + /// parameterised sweep - a copy-paste slip between them is exactly what this catches. + /// + [TestFixture] + public class UpdateAndDeletePairingTests : ShuffleTestBase + { + private const string ExecuteMultiple = "ExecuteMultiple"; + + private static Shuffler.TestUpdateBatch Updates(int count) + { + var batch = new Shuffler.TestUpdateBatch(); + for (var i = 1; i <= count; i++) + { + batch.Add(Record("account", Id(i), "Account " + i), "account Acme " + i); + } + return batch; + } + + private static List Deletes(int count) + { + var batch = new List(); + for (var i = 1; i <= count; i++) + { + batch.Add(Record("account", Id(i))); + } + return batch; + } + + [Test] + public void An_update_fault_is_one_failure_and_the_rest_still_count() + { + Service.OnMessage(ExecuteMultiple, new ExecuteMultipleResponseBuilder() + .Succeeded() + .Fault("Bad row") + .Succeeded() + .Build()); + + var outcome = NewShuffler().TestFlushUpdatesWithExecuteMultiple(Updates(3)); + + Assert.That(outcome.Updated, Is.EqualTo(2), outcome.ToString()); + Assert.That(outcome.Failed, Is.EqualTo(1), outcome.ToString()); + Recorder.AssertSent("002 Update Failed: account Acme 2 account Bad row"); + Recorder.AssertSent("003 Updated: account Acme 3"); + } + + [Test] + public void Unanswered_updates_are_reported_as_not_executed() + { + Service.OnMessage(ExecuteMultiple, new ExecuteMultipleResponseBuilder() + .Succeeded() + .Omit() + .Omit() + .Build()); + + var outcome = NewShuffler().TestFlushUpdatesWithExecuteMultiple(Updates(3)); + + Assert.That(outcome.Updated, Is.EqualTo(1), outcome.ToString()); + Assert.That(outcome.Failed, Is.EqualTo(2), outcome.ToString()); + Recorder.AssertSent("002 Update Not Executed: account Acme 2 account"); + } + + [Test] + public void A_batch_update_that_throws_falls_back_to_one_update_per_record() + { + Service.OnMessage(ExecuteMultiple, request => + { + throw ExecuteMultipleResponseBuilder.Faulted("Batch too large"); + }); + + var outcome = NewShuffler().TestFlushUpdatesWithExecuteMultiple(Updates(3)); + + Assert.That(outcome.Updated, Is.EqualTo(3), outcome.ToString()); + Assert.That(Service.CountOf("Update"), Is.EqualTo(3)); + Recorder.AssertLogged("Falling back to sequential updates"); + } + + [Test] + public void A_delete_of_a_record_that_is_already_gone_is_not_a_failure() + { + // Deleting what is not there is the expected state, not an error - the import is + // being asked to make the record absent, and it is. + Service.OnMessage(ExecuteMultiple, new ExecuteMultipleResponseBuilder() + .Succeeded() + .Fault("account With Id = ... Does Not Exist") + .Build()); + + var outcome = NewShuffler().TestFlushPendingDeletes(Deletes(2)); + + Assert.That(outcome.Deleted, Is.EqualTo(1), outcome.ToString()); + Assert.That(outcome.Failed, Is.EqualTo(0), outcome.ToString()); + Recorder.AssertSent(" ...already deleted"); + } + + [Test] + public void A_real_delete_fault_is_counted_and_named() + { + Service.OnMessage(ExecuteMultiple, new ExecuteMultipleResponseBuilder() + .Succeeded() + .Fault("Cannot delete, still referenced") + .Build()); + + var outcome = NewShuffler().TestFlushPendingDeletes(Deletes(2)); + + Assert.That(outcome.Deleted, Is.EqualTo(1), outcome.ToString()); + Assert.That(outcome.Failed, Is.EqualTo(1), outcome.ToString()); + Recorder.AssertSent("Delete Failed: account Cannot delete, still referenced"); + } + + [Test] + public void Unanswered_deletes_are_reported_as_not_executed() + { + Service.OnMessage(ExecuteMultiple, new ExecuteMultipleResponseBuilder() + .Succeeded() + .Omit() + .Build()); + + var outcome = NewShuffler().TestFlushPendingDeletes(Deletes(2)); + + Assert.That(outcome.Deleted, Is.EqualTo(1), outcome.ToString()); + Assert.That(outcome.Failed, Is.EqualTo(1), outcome.ToString()); + Recorder.AssertSent("Delete Not Executed: account"); + } + + [Test] + public void A_batch_delete_that_throws_falls_back_to_one_delete_per_record() + { + Service.OnMessage(ExecuteMultiple, request => + { + throw ExecuteMultipleResponseBuilder.Faulted("Batch too large"); + }); + + var outcome = NewShuffler().TestFlushPendingDeletes(Deletes(3)); + + Assert.That(outcome.Deleted, Is.EqualTo(3), outcome.ToString()); + Assert.That(Service.CountOf("Delete"), Is.EqualTo(3)); + Recorder.AssertLogged("Falling back to sequential deletes"); + } + + [Test] + public void A_single_record_batch_never_reaches_ExecuteMultiple() + { + // Every dispatcher short-circuits at one record. A fixture that forgets this + // scripts a batch response that is never asked for. + var outcome = NewShuffler().TestFlushPendingDeletes(Deletes(1)); + + Assert.That(outcome.Deleted, Is.EqualTo(1)); + Assert.That(Service.CountOf(ExecuteMultiple), Is.EqualTo(0)); + Assert.That(Service.CountOf("Delete"), Is.EqualTo(1)); + } + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Layer2/BatchabilityTests.cs b/tests/Xrm.Shuffle.Core.Tests/Layer2/BatchabilityTests.cs new file mode 100644 index 0000000..36afa5c --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Layer2/BatchabilityTests.cs @@ -0,0 +1,138 @@ +namespace Cinteros.Crm.Utils.Shuffle.Tests.Layer2 +{ + using System; + using System.Linq; + using Cinteros.Crm.Utils.Shuffle.Tests.Helpers; + using Microsoft.Xrm.Sdk; + using NUnit.Framework; + + /// + /// Records carrying state or owner leave the batch, and take the batch's ordering with them. + /// + /// + /// IsBatchable rejects any record holding statecode, statuscode or + /// ownerid, because none of those can be expressed as a plain create or update - they + /// need a SetState or an Assign afterwards. The consequence worth pinning is not that the + /// record is excluded, but that reaching it flushes everything queued before it: the product + /// has to keep the definition's record order, and a batch held open across a non-batchable + /// record would not. + /// + [TestFixture] + public class BatchabilityTests : FakeOrgTestBase + { + private static Types.DataBlock CreateBlock() + { + return DefinitionXml.DataBlock("Accounts", "account") + .BatchSize(50) + .DeserializeBlock(); + } + + private static Entity WithOwner(string name, Guid owner) + { + var entity = Record("account", Id(name.Length + owner.GetHashCode()), name); + entity["ownerid"] = new EntityReference("systemuser", owner); + return entity; + } + + private static EntityCollection Collection(params Entity[] entities) + { + var collection = new EntityCollection { EntityName = "account" }; + foreach (var entity in entities) + { + collection.Entities.Add(entity); + } + return collection; + } + + [Test] + public void A_record_with_an_owner_is_created_on_its_own() + { + Online().WithMetadata("account"); + + var outcome = NewShuffler().TestImportDataBlock( + CreateBlock(), + Collection( + Record("account", Id(1), "Alpha"), + WithOwner("Beta", Id(90)), + Record("account", Id(3), "Gamma"))); + + Assert.That(outcome.Created, Is.EqualTo(3), DumpAll()); + Assert.That(Org.Rows("account").Count, Is.EqualTo(3), DumpAll()); + } + + /// + /// Two batched records around one that is not batchable have to come out as two separate + /// CreateMultiple calls, not one - otherwise the middle record would be written after + /// both of them. + /// + [Test] + public void The_batch_is_flushed_before_the_non_batchable_record() + { + Online().WithMetadata("account"); + + NewShuffler().TestImportDataBlock( + CreateBlock(), + Collection( + Record("account", Id(1), "Alpha"), + Record("account", Id(2), "Beta"), + WithOwner("Gamma", Id(90)), + Record("account", Id(4), "Delta"), + Record("account", Id(5), "Epsilon"))); + + Assert.That(Service.CountOf("CreateMultiple"), Is.EqualTo(2), DumpAll()); + + var first = RecordingOrganizationService.TargetsOf( + Service.RequestsNamed("CreateMultiple")[0]); + Assert.That(first.Count, Is.EqualTo(2), DumpAll()); + Assert.That(first[0]["name"], Is.EqualTo("Alpha"), DumpAll()); + Assert.That(first[1]["name"], Is.EqualTo("Beta"), DumpAll()); + } + + /// + /// The interleaving, read off the request stream: batch, single, batch. Anything else + /// means a record was written out of the order the definition listed it in. + /// + [Test] + public void The_single_record_is_written_between_the_two_batches() + { + Online().WithMetadata("account"); + + NewShuffler().TestImportDataBlock( + CreateBlock(), + Collection( + Record("account", Id(1), "Alpha"), + Record("account", Id(2), "Beta"), + WithOwner("Gamma", Id(90)), + Record("account", Id(4), "Delta"), + Record("account", Id(5), "Epsilon"))); + + var writes = Service.RequestNames + .Where(name => name == "Create" || name == "CreateMultiple") + .ToList(); + Assert.That( + writes, + Is.EqualTo(new[] { "CreateMultiple", "Create", "CreateMultiple" }), + DumpAll()); + } + + /// + /// A block made entirely of non-batchable records sends no bulk message at all, which is + /// the same shape as BatchSize being left at its default. + /// + [Test] + public void A_block_of_owned_records_never_reaches_a_bulk_message() + { + Online().WithMetadata("account"); + + var outcome = NewShuffler().TestImportDataBlock( + CreateBlock(), + Collection( + WithOwner("Alpha", Id(90)), + WithOwner("Beta", Id(91)))); + + Assert.That(outcome.Created, Is.EqualTo(2), DumpAll()); + Assert.That(Service.CountOf("CreateMultiple"), Is.EqualTo(0), DumpAll()); + Assert.That(Service.CountOf("ExecuteMultiple"), Is.EqualTo(0), DumpAll()); + } + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Layer2/CapabilityDetectionTests.cs b/tests/Xrm.Shuffle.Core.Tests/Layer2/CapabilityDetectionTests.cs new file mode 100644 index 0000000..b1aadf4 --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Layer2/CapabilityDetectionTests.cs @@ -0,0 +1,111 @@ +namespace Cinteros.Crm.Utils.Shuffle.Tests.Layer2 +{ + using System.Linq; + using Cinteros.Crm.Utils.Shuffle.Tests.Helpers; + using Microsoft.Xrm.Sdk; + using NUnit.Framework; + + /// + /// Which message a batch goes out as, decided by what the organization says it supports. + /// + /// + /// The product asks sdkmessagefilter once per entity and message, caches the answer + /// for the run, and silently caches false if the query throws. That makes an + /// unasserted routing decision worthless - a test that only counted records would pass + /// against the fallback path just as happily. Every test here names the message it expects. + /// + [TestFixture] + public class CapabilityDetectionTests : FakeOrgTestBase + { + private static EntityCollection TwoAccounts() + { + var entities = new EntityCollection { EntityName = "account" }; + entities.Entities.Add(Record("account", Id(1), "Alpha")); + entities.Entities.Add(Record("account", Id(2), "Beta")); + return entities; + } + + private static Types.DataBlock CreateBlock(int batchSize) + { + return DefinitionXml.DataBlock("Accounts", "account") + .BatchSize(batchSize) + .DeserializeBlock(); + } + + [Test] + public void An_org_that_supports_CreateMultiple_gets_one_CreateMultiple() + { + Online().WithMetadata("account"); + + var outcome = NewShuffler().TestImportDataBlock(CreateBlock(10), TwoAccounts()); + + Assert.That(outcome.Created, Is.EqualTo(2), DumpAll()); + Assert.That(Service.CountOf("CreateMultiple"), Is.EqualTo(1), DumpAll()); + Assert.That(Service.CountOf("ExecuteMultiple"), Is.EqualTo(0), DumpAll()); + Assert.That(Org.Rows("account").Count, Is.EqualTo(2), DumpAll()); + } + + [Test] + public void An_org_without_CreateMultiple_falls_back_to_ExecuteMultiple() + { + OnPrem().WithMetadata("account"); + + var outcome = NewShuffler().TestImportDataBlock(CreateBlock(10), TwoAccounts()); + + Assert.That(outcome.Created, Is.EqualTo(2), DumpAll()); + Assert.That(Service.CountOf("CreateMultiple"), Is.EqualTo(0), DumpAll()); + Assert.That(Service.CountOf("ExecuteMultiple"), Is.EqualTo(1), DumpAll()); + Assert.That(Org.Rows("account").Count, Is.EqualTo(2), DumpAll()); + } + + [Test] + public void The_probe_result_is_cached_for_the_whole_run() + { + Online().WithMetadata("account"); + + var shuffler = NewShuffler(); + shuffler.TestImportDataBlock(CreateBlock(2), TwoAccounts()); + shuffler.TestImportDataBlock(CreateBlock(2), TwoAccounts()); + + Assert.That(Service.CountOf("CreateMultiple"), Is.EqualTo(2), DumpAll()); + Assert.That( + Org.Logger.CountLogged("CreateMultiple support for account"), + Is.EqualTo(1), + DumpAll()); + } + + [Test] + public void The_probe_announces_the_answer_it_cached() + { + OnPrem().WithMetadata("account"); + + NewShuffler().TestImportDataBlock(CreateBlock(10), TwoAccounts()); + + Assert.That(Org.Logger.Logged("CreateMultiple support for account: False"), DumpAll()); + } + + /// + /// A landmine marker, not a requirement. sdkmessagefilter.primaryobjecttypecode + /// holds a numeric entity type code on a real platform, but the product queries it with + /// the logical name, so the seed in matches the product + /// rather than the platform. If the product is ever fixed to query by type code, this + /// test fails and points at both places that have to change together. + /// + [Test] + public void Capability_query_uses_the_logical_name_not_the_entity_type_code() + { + Online().WithMetadata("account"); + + NewShuffler().TestImportDataBlock(CreateBlock(10), TwoAccounts()); + + var probe = Service.Queries + .OfType() + .FirstOrDefault(q => q.EntityName == "sdkmessagefilter"); + + Assert.That(probe, Is.Not.Null, DumpAll()); + var condition = probe.Criteria.Conditions + .First(c => c.AttributeName == "primaryobjecttypecode"); + Assert.That(condition.Values.Single(), Is.EqualTo("account"), DumpAll()); + } + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Layer2/ExportAttributeSelectionTests.cs b/tests/Xrm.Shuffle.Core.Tests/Layer2/ExportAttributeSelectionTests.cs new file mode 100644 index 0000000..836e36b --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Layer2/ExportAttributeSelectionTests.cs @@ -0,0 +1,208 @@ +namespace Cinteros.Crm.Utils.Shuffle.Tests.Layer2 +{ + using System; + using System.Collections.Generic; + using System.Linq; + using Cinteros.Crm.Utils.Shuffle.Tests.Helpers; + using Microsoft.Xrm.Sdk; + using NUnit.Framework; + + /// + /// Which attributes survive an export that asked for a wildcard column set. + /// + /// + /// + /// A block with a wildcard column reads every attribute back from the platform and then + /// narrows the result in memory, because a QueryExpression cannot express "cint_*". That + /// narrowing is SelectAttributes, and it is the only place in export where records are + /// edited after they are retrieved - what it drops never reaches the file. + /// + /// + /// It was rewritten in this PR. The previous version removed keys from the same collection + /// it was walking, which shifted the remaining entries down and skipped the one after every + /// removal, so a run of unwanted attributes came out half-filtered. The two-pass version + /// collects the keys first and removes afterwards. Several cases below only distinguish the + /// two when unwanted attributes are adjacent, which is why they are written that way. + /// + /// + [TestFixture] + public class ExportAttributeSelectionTests : FakeOrgTestBase + { + /// A retrieved record, carrying its primary id the way a real retrieve answers. + private static Entity Row(Guid id, params string[] attributes) + { + var entity = new Entity("account", id); + entity["accountid"] = id; + foreach (var attribute in attributes) + { + entity[attribute] = attribute + " value"; + } + return entity; + } + + private static EntityCollection Exported(params Entity[] entities) + { + var collection = new EntityCollection { EntityName = "account" }; + collection.Entities.AddRange(entities); + return collection; + } + + /// + /// Narrows a retrieved collection the way an export block with a wildcard column does. + /// + private void Select(EntityCollection exported, string[] keep, string[] nulls = null) + { + Online().WithMetadata("account"); + Shuffler.TestSelectAttributes( + Org.Container, + exported, + keep.ToList(), + (nulls ?? new string[0]).ToList()); + } + + private static List Attributes(Entity entity) + { + return entity.Attributes.Keys.OrderBy(k => k, StringComparer.Ordinal).ToList(); + } + + [Test] + public void An_attribute_outside_the_list_is_removed() + { + var record = Row(Id(1), "name", "telephone1"); + + Select(Exported(record), new[] { "name" }); + + Assert.That(Attributes(record), Is.EqualTo(new[] { "accountid", "name" }), DumpAll()); + } + + /// + /// The regression the rewrite fixes: four unwanted attributes in a row. + /// + /// + /// Removing while iterating dropped every other one, so this record used to come out + /// still carrying two of the four. + /// + [Test] + public void A_run_of_unwanted_attributes_is_removed_entirely() + { + var record = Row(Id(1), "name", "junk1", "junk2", "junk3", "junk4"); + + Select(Exported(record), new[] { "name" }); + + Assert.That(Attributes(record), Is.EqualTo(new[] { "accountid", "name" }), DumpAll()); + } + + [Test] + public void An_unwanted_attribute_between_two_wanted_ones_is_removed() + { + var record = Row(Id(1), "name", "junk1", "description", "junk2"); + + Select(Exported(record), new[] { "name", "description" }); + + Assert.That( + Attributes(record), + Is.EqualTo(new[] { "accountid", "description", "name" }), + DumpAll()); + } + + [Test] + public void The_primary_id_survives_a_list_that_does_not_name_it() + { + var record = Row(Id(1), "name"); + + Select(Exported(record), new[] { "name" }); + + Assert.That(record.Contains("accountid"), Is.True, DumpAll()); + Assert.That(record["accountid"], Is.EqualTo(Id(1)), DumpAll()); + } + + [Test] + public void An_empty_list_strips_everything_but_the_primary_id() + { + var record = Row(Id(1), "name", "telephone1", "description"); + + Select(Exported(record), new string[0]); + + Assert.That(Attributes(record), Is.EqualTo(new[] { "accountid" }), DumpAll()); + } + + [Test] + public void A_trailing_wildcard_keeps_every_attribute_with_that_prefix() + { + var record = Row(Id(1), "cint_one", "cint_two", "name", "telephone1"); + + Select(Exported(record), new[] { "cint_*" }); + + Assert.That( + Attributes(record), + Is.EqualTo(new[] { "accountid", "cint_one", "cint_two" }), + DumpAll()); + } + + /// + /// Definitions write the wildcard as a SQL percent sign, which IsSqlLikeMatch rewrites + /// to an asterisk - so both spellings have to select the same columns. + /// + [Test] + public void A_percent_wildcard_selects_the_same_attributes_as_an_asterisk() + { + var record = Row(Id(1), "cint_one", "cint_two", "name"); + + Select(Exported(record), new[] { "cint_%" }); + + Assert.That( + Attributes(record), + Is.EqualTo(new[] { "accountid", "cint_one", "cint_two" }), + DumpAll()); + } + + [Test] + public void A_null_attribute_is_added_when_the_record_does_not_carry_it() + { + var record = Row(Id(1), "name"); + + Select(Exported(record), new[] { "name" }, new[] { "description" }); + + Assert.That(record.Contains("description"), Is.True, DumpAll()); + Assert.That(record["description"], Is.Null, DumpAll()); + } + + [Test] + public void A_null_attribute_the_record_already_carries_keeps_its_value() + { + var record = Row(Id(1), "description"); + + Select(Exported(record), new[] { "description" }, new[] { "description" }); + + Assert.That(record["description"], Is.EqualTo("description value"), DumpAll()); + } + + /// + /// Null attributes are applied after the filter, not before, so a column the filter + /// dropped comes back - emptied. Worth pinning: it is the difference between a file + /// that omits a column and one that blanks it on import. + /// + [Test] + public void A_null_attribute_the_filter_removed_comes_back_empty() + { + var record = Row(Id(1), "name", "description"); + + Select(Exported(record), new[] { "name" }, new[] { "description" }); + + Assert.That(record.Contains("description"), Is.True, DumpAll()); + Assert.That(record["description"], Is.Null, DumpAll()); + } + + [Test] + public void Every_record_in_the_collection_is_filtered() + { + var first = Row(Id(1), "name", "junk1", "junk2"); + var second = Row(Id(2), "name", "junk1", "junk2"); + + Select(Exported(first, second), new[] { "name" }); + + Assert.That(Attributes(first), Is.EqualTo(new[] { "accountid", "name" }), DumpAll()); + Assert.That(Attributes(second), Is.EqualTo(new[] { "accountid", "name" }), DumpAll()); + } + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Layer2/MatchResolutionTests.cs b/tests/Xrm.Shuffle.Core.Tests/Layer2/MatchResolutionTests.cs new file mode 100644 index 0000000..f52135c --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Layer2/MatchResolutionTests.cs @@ -0,0 +1,153 @@ +namespace Cinteros.Crm.Utils.Shuffle.Tests.Layer2 +{ + using System.Linq; + using Cinteros.Crm.Utils.Shuffle.Tests.Helpers; + using Microsoft.Xrm.Sdk; + using NUnit.Framework; + + /// + /// What a match attribute resolves to, and what the block does with each answer. + /// + /// + /// Three answers are possible and all three are reachable from the same definition: no + /// match creates, one match updates, several matches fail the record without touching the + /// organization. The interesting part is that the first two go into a batch while the third + /// does not, so a block of mixed records ends up with fewer batched rows than it read. + /// + [TestFixture] + public class MatchResolutionTests : FakeOrgTestBase + { + private static Types.DataBlock MatchOnName() + { + return DefinitionXml.DataBlock("Accounts", "account") + .BatchSize(10) + .ImportAttribute("UpdateIdentical", "true") + .MatchOn("name") + .DeserializeBlock(); + } + + private static EntityCollection Sources(params string[] names) + { + var entities = new EntityCollection { EntityName = "account" }; + var seed = 1; + foreach (var name in names) + { + entities.Entities.Add(Record("account", Id(seed++), name)); + } + return entities; + } + + [Test] + public void No_match_creates_the_record() + { + Online().WithMetadata("account"); + + var outcome = NewShuffler().TestImportDataBlock(MatchOnName(), Sources("Alpha", "Beta")); + + Assert.That(outcome.Created, Is.EqualTo(2), DumpAll()); + Assert.That(outcome.Updated, Is.EqualTo(0), DumpAll()); + Assert.That(Service.CountOf("CreateMultiple"), Is.EqualTo(1), DumpAll()); + Assert.That(Service.CountOf("UpdateMultiple"), Is.EqualTo(0), DumpAll()); + } + + [Test] + public void One_match_updates_it_in_place() + { + Online() + .WithEntity( + Seeded("account", Id(101), "Alpha"), + Seeded("account", Id(102), "Beta")); + + var outcome = NewShuffler().TestImportDataBlock(MatchOnName(), Sources("Alpha", "Beta")); + + Assert.That(outcome.Updated, Is.EqualTo(2), DumpAll()); + Assert.That(outcome.Created, Is.EqualTo(0), DumpAll()); + Assert.That(Service.CountOf("UpdateMultiple"), Is.EqualTo(1), DumpAll()); + Assert.That(Org.Rows("account").Count, Is.EqualTo(2), "no row should have been added"); + } + + [Test] + public void An_update_is_written_against_the_matched_id_not_the_source_id() + { + // Two matching records, because a batch of one is flushed as a plain Update and the + // request this asserts on would never be sent. + Online() + .WithEntity( + Seeded("account", Id(101), "Alpha"), + Seeded("account", Id(102), "Beta")); + + NewShuffler().TestImportDataBlock(MatchOnName(), Sources("Alpha", "Beta")); + + var targets = RecordingOrganizationService.TargetsOf( + Service.RequestsNamed("UpdateMultiple")[0]); + Assert.That( + targets.Select(t => t.Id).ToList(), + Is.EqualTo(new[] { Id(101), Id(102) }), + DumpAll()); + Assert.That(targets.Select(t => t.Id), Has.No.Member(Id(1)), DumpAll()); + } + + [Test] + public void Several_matches_fail_the_record_and_write_nothing() + { + Online() + .WithEntity( + Seeded("account", Id(101), "Alpha"), + Seeded("account", Id(102), "Alpha")); + + var outcome = NewShuffler().TestImportDataBlock(MatchOnName(), Sources("Alpha")); + + Assert.That(outcome.Failed, Is.EqualTo(1), DumpAll()); + Assert.That(outcome.Accounted, Is.EqualTo(1), DumpAll()); + Assert.That( + Org.Logger.Logged("001 Match Failed: Alpha matches 2 records in target database"), + DumpAll()); + Assert.That(Service.CountOf("UpdateMultiple"), Is.EqualTo(0), DumpAll()); + Assert.That(Service.CountOf("CreateMultiple"), Is.EqualTo(0), DumpAll()); + } + + [Test] + public void An_ambiguous_record_does_not_stop_the_rest_of_the_block() + { + Online() + .WithEntity( + Seeded("account", Id(101), "Alpha"), + Seeded("account", Id(102), "Alpha"), + Seeded("account", Id(103), "Beta")); + + var outcome = NewShuffler().TestImportDataBlock( + MatchOnName(), Sources("Alpha", "Beta", "Gamma")); + + Assert.That(outcome.Failed, Is.EqualTo(1), DumpAll()); + Assert.That(outcome.Updated, Is.EqualTo(1), DumpAll()); + Assert.That(outcome.Created, Is.EqualTo(1), DumpAll()); + Assert.That(outcome.Accounted, Is.EqualTo(3), DumpAll()); + } + + /// + /// PreRetrieveAll takes one snapshot for the whole block, which is the property that + /// lets a matched block batch at all - a per-record match query would have to see the + /// rows the batch has not written yet. + /// + [Test] + public void PreRetrieveAll_reads_the_target_entity_once_for_the_whole_block() + { + Online().WithEntity( + Seeded("account", Id(101), "Alpha"), + Seeded("account", Id(102), "Beta")); + + NewShuffler().TestImportDataBlock(MatchOnName(), Sources("Alpha", "Beta")); + + var accountQueries = 0; + foreach (var query in Service.Queries) + { + var expression = query as Microsoft.Xrm.Sdk.Query.QueryExpression; + if (expression != null && expression.EntityName == "account") + { + accountQueries++; + } + } + Assert.That(accountQueries, Is.EqualTo(1), DumpAll()); + } + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Layer2/SkipIdenticalTests.cs b/tests/Xrm.Shuffle.Core.Tests/Layer2/SkipIdenticalTests.cs new file mode 100644 index 0000000..c1dc853 --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Layer2/SkipIdenticalTests.cs @@ -0,0 +1,169 @@ +namespace Cinteros.Crm.Utils.Shuffle.Tests.Layer2 +{ + using System; + using System.Linq; + using Cinteros.Crm.Utils.Shuffle.Tests.Helpers; + using Microsoft.Xrm.Sdk; + using NUnit.Framework; + + /// + /// A matched record that already holds the source values is skipped rather than batched. + /// + /// + /// This is the one decision the batched path makes per record that has nothing to do with + /// batching, and it is the easiest to break: the comparison reads the source record's own + /// attribute keys, minus the primary id, so widening a block with one extra column changes + /// which records count as identical. The counters are what a definition author sees, so the + /// tests assert Skipped and Updated, not just the request stream. + /// + [TestFixture] + public class SkipIdenticalTests : FakeOrgTestBase + { + private static Types.DataBlock MatchOnName(bool updateIdentical = false) + { + var block = DefinitionXml.DataBlock("Accounts", "account").BatchSize(10); + if (updateIdentical) + { + block = block.ImportAttribute("UpdateIdentical", "true"); + } + return block.MatchOn("name").DeserializeBlock(); + } + + private static Entity Source(Guid id, string name, string city = null) + { + var entity = new Entity("account", id); + entity["name"] = name; + if (city != null) + { + entity["address1_city"] = city; + } + return entity; + } + + private static EntityCollection Collection(params Entity[] entities) + { + var collection = new EntityCollection { EntityName = "account" }; + foreach (var entity in entities) + { + collection.Entities.Add(entity); + } + return collection; + } + + [Test] + public void An_identical_match_is_skipped_and_nothing_is_written() + { + Online().WithEntity( + Seeded("account", Id(101), "Alpha"), + Seeded("account", Id(102), "Beta")); + + var outcome = NewShuffler().TestImportDataBlock( + MatchOnName(), + Collection(Source(Id(1), "Alpha"), Source(Id(2), "Beta"))); + + Assert.That(outcome.Skipped, Is.EqualTo(2), DumpAll()); + Assert.That(outcome.Updated, Is.EqualTo(0), DumpAll()); + Assert.That(Service.CountOf("UpdateMultiple"), Is.EqualTo(0), DumpAll()); + Assert.That( + Org.Logger.Logged("001 Skipped: Alpha (Identical)"), + DumpAll()); + } + + [Test] + public void A_differing_attribute_is_batched_as_an_update() + { + var existing = Seeded("account", Id(101), "Alpha"); + existing["address1_city"] = "Stockholm"; + var untouched = Seeded("account", Id(102), "Beta"); + untouched["address1_city"] = "Uppsala"; + Online().WithEntity(existing, untouched); + + var outcome = NewShuffler().TestImportDataBlock( + MatchOnName(), + Collection( + Source(Id(1), "Alpha", "Gothenburg"), + Source(Id(2), "Beta", "Uppsala"))); + + Assert.That(outcome.Updated, Is.EqualTo(1), DumpAll()); + Assert.That(outcome.Skipped, Is.EqualTo(1), DumpAll()); + Assert.That(Service.CountOf("UpdateMultiple"), Is.EqualTo(0), "one row is not a batch"); + Assert.That(Service.Updated.Count, Is.EqualTo(1), DumpAll()); + Assert.That(Service.Updated[0].Id, Is.EqualTo(Id(101)), DumpAll()); + } + + /// + /// UpdateIdentical turns the comparison off wholesale, which is also the setting that + /// opens the upsert gate - so a block that sets it writes every matched record whether + /// anything changed or not. + /// + [Test] + public void UpdateIdentical_writes_the_record_anyway() + { + Online().WithEntity( + Seeded("account", Id(101), "Alpha"), + Seeded("account", Id(102), "Beta")); + + var outcome = NewShuffler().TestImportDataBlock( + MatchOnName(true), + Collection(Source(Id(1), "Alpha"), Source(Id(2), "Beta"))); + + Assert.That(outcome.Updated, Is.EqualTo(2), DumpAll()); + Assert.That(outcome.Skipped, Is.EqualTo(0), DumpAll()); + Assert.That(Service.CountOf("UpdateMultiple"), Is.EqualTo(1), DumpAll()); + } + + /// + /// The comparison walks the source record's keys, not the target's, so an attribute the + /// target holds and the source does not cannot make the two differ. + /// + [Test] + public void An_attribute_only_the_target_holds_does_not_count_as_a_difference() + { + var existing = Seeded("account", Id(101), "Alpha"); + existing["address1_city"] = "Stockholm"; + var second = Seeded("account", Id(102), "Beta"); + second["address1_city"] = "Uppsala"; + Online().WithEntity(existing, second); + + var outcome = NewShuffler().TestImportDataBlock( + MatchOnName(), + Collection(Source(Id(1), "Alpha"), Source(Id(2), "Beta"))); + + Assert.That(outcome.Skipped, Is.EqualTo(2), DumpAll()); + Assert.That(outcome.Updated, Is.EqualTo(0), DumpAll()); + } + + /// + /// Skipping does not cost the record its guid mapping - later blocks still have to be + /// able to point an EntityReference at it. + /// + [Test] + public void A_skipped_record_is_still_mapped_from_its_source_id() + { + Online().WithEntity(Seeded("account", Id(101), "Alpha")); + + var shuffler = NewShuffler(); + shuffler.TestImportDataBlock( + MatchOnName(), + Collection(Source(Id(1), "Alpha"), Source(Id(2), "Beta"))); + + Assert.That(shuffler.TestGuidMap.ContainsKey(Id(1)), DumpAll()); + Assert.That(shuffler.TestGuidMap[Id(1)], Is.EqualTo(Id(101)), DumpAll()); + } + + [Test] + public void A_skipped_record_is_not_reported_as_a_reference() + { + Online().WithEntity(Seeded("account", Id(101), "Alpha")); + + var outcome = NewShuffler().TestImportDataBlock( + MatchOnName(), + Collection(Source(Id(1), "Alpha"), Source(Id(2), "Beta"))); + + Assert.That( + outcome.References.Any(reference => reference.Id == Id(101)), + Is.False, + DumpAll()); + } + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Layer2/UpsertGateTests.cs b/tests/Xrm.Shuffle.Core.Tests/Layer2/UpsertGateTests.cs new file mode 100644 index 0000000..7cc8a97 --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Layer2/UpsertGateTests.cs @@ -0,0 +1,157 @@ +namespace Cinteros.Crm.Utils.Shuffle.Tests.Layer2 +{ + using Cinteros.Crm.Utils.Shuffle.Tests.Helpers; + using Microsoft.Xrm.Sdk; + using NUnit.Framework; + + /// + /// The five conditions that together decide whether a block upserts instead of matching. + /// + /// + /// Upsert skips the pre-retrieval query entirely, which is a large behaviour change for a + /// block that asked for none of it. The gate is one && chain of five clauses - + /// Save is CreateUpdate, CreateWithId is set, there is at least one match attribute, Delete + /// is None, and UpdateIdentical is set - so each test here opens the gate and then breaks + /// exactly one clause. Two of them are off by default, which is why the opt-in test has to + /// set them explicitly. + /// + [TestFixture] + public class UpsertGateTests : FakeOrgTestBase + { + /// + /// Every clause of the upsert gate met at once. is a + /// parameter rather than a later override because the builder appends attributes instead + /// of replacing them, so setting one twice emits duplicate XML and the document will not + /// load. + /// + private static DefinitionXml.DataBlockBuilder GateOpen(bool createWithId = true) + { + return DefinitionXml.DataBlock("Accounts", "account") + .BatchSize(10) + .CreateWithId(createWithId) + .ImportAttribute("UpdateIdentical", "true") + .MatchOn("name"); + } + + private static EntityCollection TwoAccounts() + { + var entities = new EntityCollection { EntityName = "account" }; + entities.Entities.Add(Record("account", Id(1), "Alpha")); + entities.Entities.Add(Record("account", Id(2), "Beta")); + return entities; + } + + private void Run(DefinitionXml.DataBlockBuilder block) + { + Online().WithMetadata("account"); + NewShuffler().TestImportDataBlock(block.DeserializeBlock(), TwoAccounts()); + } + + /// + /// How many times the target entity itself was queried. The capability probe queries + /// sdkmessagefilter, so counting all queries would never reach zero. + /// + private int AccountQueries() + { + var count = 0; + foreach (var query in Service.Queries) + { + var expression = query as Microsoft.Xrm.Sdk.Query.QueryExpression; + if (expression != null && expression.EntityName == "account") + { + count++; + } + } + return count; + } + + private void AssertGateClosed() + { + Assert.That(Service.CountOf("UpsertMultiple"), Is.EqualTo(0), DumpAll()); + Assert.That( + Org.Logger.Logged("Upsert path enabled"), + Is.False, + DumpAll()); + } + + [Test] + public void All_five_clauses_met_upserts_the_block() + { + Run(GateOpen()); + + Assert.That(Service.CountOf("UpsertMultiple"), Is.EqualTo(1), DumpAll()); + Assert.That( + Org.Logger.Logged( + "Upsert path enabled - records will be upserted without pre-retrieval queries"), + DumpAll()); + } + + /// + /// PreRetrieveAll is the switch that made the block batchable in the first place, so the + /// product says out loud that it is ignoring it rather than leaving the reader to wonder + /// why no query went out. + /// + [Test] + public void An_upserting_block_says_it_is_skipping_PreRetrieveAll() + { + Run(GateOpen()); + + Assert.That( + Org.Logger.Logged( + "Note: PreRetrieveAll is not needed when using Upsert and will be skipped"), + DumpAll()); + Assert.That(AccountQueries(), Is.EqualTo(0), DumpAll()); + } + + [Test] + public void Save_other_than_CreateUpdate_closes_the_gate() + { + Run(GateOpen().ImportAttribute("Save", "CreateOnly")); + + AssertGateClosed(); + } + + [Test] + public void Without_CreateWithId_the_gate_stays_closed() + { + Run(GateOpen(createWithId: false)); + + AssertGateClosed(); + } + + [Test] + public void Without_a_match_attribute_the_gate_stays_closed() + { + Run(DefinitionXml.DataBlock("Accounts", "account") + .BatchSize(10) + .CreateWithId(true) + .ImportAttribute("UpdateIdentical", "true")); + + AssertGateClosed(); + Assert.That(Service.CountOf("CreateMultiple"), Is.EqualTo(1), DumpAll()); + } + + [Test] + public void Any_delete_setting_closes_the_gate() + { + Run(GateOpen().ImportAttribute("Delete", "Existing")); + + AssertGateClosed(); + } + + /// + /// UpdateIdentical defaults to false, so a definition that sets only CreateWithId and a + /// match attribute does not silently become an upserting block. + /// + [Test] + public void The_default_UpdateIdentical_keeps_the_gate_closed() + { + Run(DefinitionXml.DataBlock("Accounts", "account") + .BatchSize(10) + .CreateWithId(true) + .MatchOn("name")); + + AssertGateClosed(); + } + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Layer3/DeferStateAndOwnerTests.cs b/tests/Xrm.Shuffle.Core.Tests/Layer3/DeferStateAndOwnerTests.cs new file mode 100644 index 0000000..4241d92 --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Layer3/DeferStateAndOwnerTests.cs @@ -0,0 +1,477 @@ +namespace Cinteros.Crm.Utils.Shuffle.Tests.Layer3 +{ + using System; + using System.Linq; + using System.ServiceModel; + using Cinteros.Crm.Utils.Shuffle.Tests.Helpers; + using Microsoft.Xrm.Sdk; + using NUnit.Framework; + + /// + /// The second pass that makes state and owner batchable at all. + /// + /// + /// + /// A record carrying statecode, statuscode or ownerid cannot go in a batch - IsBatchable + /// says so, and the bulk messages would drop the values on the floor. DeferStateAndOwner + /// strips those three attributes off every record, batches what is left, and applies the + /// stripped values afterwards against the ids the first pass actually wrote. + /// + /// + /// That split is where the interesting failures live: a value queued against a source id + /// that never became a real id, a state pass that has UpdateMultiple and an owner pass that + /// never does, and a fluent Assign helper that cannot report a failure. + /// + /// + [TestFixture] + public class DeferStateAndOwnerTests : FakeOrgTestBase + { + private static Entity Stateful(string entityLogicalName, Guid id, int state, int status) + { + var entity = Record(entityLogicalName, id); + entity["statecode"] = new OptionSetValue(state); + entity["statuscode"] = new OptionSetValue(status); + return entity; + } + + private static EntityReference Owner(int seed) + { + return new EntityReference("systemuser", Id(seed)); + } + + private static OptionSetValue Option(OrganizationRequest request, string parameter) + { + return (OptionSetValue)request[parameter]; + } + + // ---- the strip pass ------------------------------------------------- + + [Test] + public void State_and_owner_are_stripped_off_the_record_and_queued() + { + OnPrem().WithMetadata("account"); + var record = Stateful("account", Id(1), 1, 2); + record["ownerid"] = Owner(9); + + var shuffler = NewShuffler(); + shuffler.TestStripAndDeferStateOwner(record); + + Assert.That(record.Contains("statecode"), Is.False, "statecode should have been stripped"); + Assert.That(record.Contains("statuscode"), Is.False, "statuscode should have been stripped"); + Assert.That(record.Contains("ownerid"), Is.False, "ownerid should have been stripped"); + Assert.That(record.Contains("name"), Is.True, "everything else should be left alone"); + Assert.That(shuffler.TestDeferredStateCount, Is.EqualTo(1)); + Assert.That(shuffler.TestDeferredOwnerCount, Is.EqualTo(1)); + Assert.That(shuffler.TestDeferredStateCodes(Id(1)), Is.EqualTo(Tuple.Create(1, 2))); + } + + /// + /// Stripping a record that carries nothing else would leave an empty record to save. + /// + [Test] + public void A_record_with_nothing_besides_state_and_owner_is_left_alone() + { + OnPrem().WithMetadata("account"); + var record = new Entity("account", Id(1)); + record["statecode"] = new OptionSetValue(1); + record["statuscode"] = new OptionSetValue(2); + + var shuffler = NewShuffler(); + shuffler.TestStripAndDeferStateOwner(record); + + Assert.That(shuffler.TestDeferredStateCount, Is.EqualTo(0)); + Assert.That(record.Contains("statecode"), Is.True, "nothing should have been stripped"); + } + + /// + /// SetState needs both halves, so a record carrying only one is not deferred - and is + /// therefore not batchable either, which is the honest outcome. + /// + [Test] + public void Statecode_without_statuscode_is_not_deferred() + { + OnPrem().WithMetadata("account"); + var record = Record("account", Id(1)); + record["statecode"] = new OptionSetValue(1); + + var shuffler = NewShuffler(); + shuffler.TestStripAndDeferStateOwner(record); + + Assert.That(shuffler.TestDeferredStateCount, Is.EqualTo(0)); + Assert.That(record.Contains("statecode"), Is.True, "a half state must not be removed"); + Assert.That(Shuffler.TestIsBatchable(record), Is.False); + } + + /// This is the whole point of the option. + [Test] + public void A_stripped_record_becomes_batchable() + { + OnPrem().WithMetadata("account"); + var record = Stateful("account", Id(1), 1, 2); + record["ownerid"] = Owner(9); + + Assert.That(Shuffler.TestIsBatchable(record), Is.False, "before"); + NewShuffler().TestStripAndDeferStateOwner(record); + Assert.That(Shuffler.TestIsBatchable(record), Is.True, "after"); + } + + // ---- filling in the real id ---------------------------------------- + + [Test] + public void The_real_id_reaches_both_queues() + { + OnPrem().WithMetadata("account"); + var shuffler = NewShuffler(); + shuffler.TestDeferState("account", Id(1), Guid.Empty, 1, 2); + shuffler.TestDeferOwner("account", Id(1), Guid.Empty, Owner(9)); + + shuffler.TestUpdateDeferredActualIds(Id(1), Id(101)); + + Assert.That(shuffler.TestDeferredActualId(Id(1)), Is.EqualTo(Id(101))); + Assert.That(shuffler.TestDeferredOwnerActualId(Id(1)), Is.EqualTo(Id(101))); + } + + [Test] + public void Only_the_matching_record_gets_the_id() + { + OnPrem().WithMetadata("account"); + var shuffler = NewShuffler(); + shuffler.TestDeferState("account", Id(1), Guid.Empty, 1, 2); + shuffler.TestDeferState("account", Id(2), Guid.Empty, 0, 1); + + shuffler.TestUpdateDeferredActualIds(Id(1), Id(101)); + + Assert.That(shuffler.TestDeferredActualId(Id(1)), Is.EqualTo(Id(101))); + Assert.That(shuffler.TestDeferredActualId(Id(2)), Is.EqualTo(Guid.Empty)); + } + + // ---- the state pass ------------------------------------------------- + + [Test] + public void Deferred_states_go_out_as_one_UpdateMultiple_when_supported() + { + Online() + .WithEntity(Seeded("account", Id(101), "Alpha"), Seeded("account", Id(102), "Beta")) + .WithAttributes("account", "statecode", "statuscode"); + var shuffler = NewShuffler(); + shuffler.TestDeferState("account", Id(1), Id(101), 1, 2, 1, "Alpha"); + shuffler.TestDeferState("account", Id(2), Id(102), 1, 2, 2, "Beta"); + + shuffler.TestFlushDeferredStateChanges(); + + Assert.That(Service.CountOf("UpdateMultiple"), Is.EqualTo(1), DumpAll()); + Assert.That(Service.CountOf("SetState"), Is.EqualTo(0), DumpAll()); + var targets = RecordingOrganizationService.TargetsOf(Service.RequestsNamed("UpdateMultiple")[0]); + Assert.That(targets.Select(t => t.Id).ToList(), Is.EqualTo(new[] { Id(101), Id(102) }), DumpAll()); + Assert.That(((OptionSetValue)targets[0]["statecode"]).Value, Is.EqualTo(1), DumpAll()); + Assert.That(((OptionSetValue)targets[0]["statuscode"]).Value, Is.EqualTo(2), DumpAll()); + Assert.That(Org.Logger.Logged("Applied 2 state changes via UpdateMultiple for account"), DumpAll()); + Assert.That(Org.Logger.Logged("Deferred state changes: 2 applied, 0 failed, 0 skipped"), DumpAll()); + } + + /// + /// The Targets collection of a bulk message names one entity, so a mixed queue cannot + /// go out in one request however many records it holds. + /// + [Test] + public void Each_entity_gets_its_own_UpdateMultiple() + { + Online() + .WithEntity( + Seeded("account", Id(101), "Alpha"), + Seeded("account", Id(102), "Beta"), + Seeded("contact", Id(201), "Carol"), + Seeded("contact", Id(202), "Dave")) + .WithAttributes("account", "statecode", "statuscode") + .WithAttributes("contact", "statecode", "statuscode"); + var shuffler = NewShuffler(); + shuffler.TestDeferState("account", Id(1), Id(101), 1, 2); + shuffler.TestDeferState("account", Id(2), Id(102), 1, 2); + shuffler.TestDeferState("contact", Id(3), Id(201), 1, 2); + shuffler.TestDeferState("contact", Id(4), Id(202), 1, 2); + + shuffler.TestFlushDeferredStateChanges(); + + Assert.That(Service.CountOf("UpdateMultiple"), Is.EqualTo(2), DumpAll()); + Assert.That( + Service.RequestsNamed("UpdateMultiple") + .Select(r => RecordingOrganizationService.TargetsOf(r).Count).ToList(), + Is.EqualTo(new[] { 2, 2 }), + DumpAll()); + Assert.That(Org.Logger.Logged("Deferred state changes: 4 applied, 0 failed, 0 skipped"), DumpAll()); + } + + [Test] + public void Without_UpdateMultiple_the_states_are_set_one_at_a_time() + { + OnPrem() + .WithEntity(Seeded("account", Id(101), "Alpha"), Seeded("account", Id(102), "Beta")) + .WithAttributes("account", "statecode", "statuscode"); + var shuffler = NewShuffler(); + shuffler.TestDeferState("account", Id(1), Id(101), 1, 2, 1, "Alpha"); + shuffler.TestDeferState("account", Id(2), Id(102), 1, 2, 2, "Beta"); + + shuffler.TestFlushDeferredStateChanges(); + + Assert.That(Service.CountOf("UpdateMultiple"), Is.EqualTo(0), DumpAll()); + Assert.That(Service.CountOf("SetState"), Is.EqualTo(2), DumpAll()); + Assert.That(Org.Logger.Logged("001 SetState (deferred): Alpha: 1/2"), DumpAll()); + Assert.That(Org.Logger.Logged("Deferred state changes: 2 applied, 0 failed, 0 skipped"), DumpAll()); + } + + /// + /// A rejected UpdateMultiple is not a failure of the records - the same states go out + /// one SetState at a time and the block still reports them applied. + /// + [Test] + public void A_failed_UpdateMultiple_falls_back_to_SetState() + { + Online() + .WithEntity(Seeded("account", Id(101), "Alpha"), Seeded("account", Id(102), "Beta")) + .WithAttributes("account", "statecode", "statuscode"); + Service.ThrowAlways("UpdateMultiple", new InvalidOperationException("no bulk here")); + var shuffler = NewShuffler(); + shuffler.TestDeferState("account", Id(1), Id(101), 1, 2); + shuffler.TestDeferState("account", Id(2), Id(102), 1, 2); + + shuffler.TestFlushDeferredStateChanges(); + + Assert.That(Service.CountOf("SetState"), Is.EqualTo(2), DumpAll()); + Assert.That(Org.Logger.Logged("UpdateMultiple for state changes failed: no bulk here"), DumpAll()); + Assert.That(Org.Logger.Logged("Deferred state changes: 2 applied, 0 failed, 0 skipped"), DumpAll()); + } + + /// + /// A record the first pass never wrote has no id to apply anything to. That is not a + /// failure - the first pass already reported why - so it is dropped and counted apart. + /// + [Test] + public void Changes_for_records_that_were_never_written_are_skipped_not_failed() + { + OnPrem() + .WithEntity(Seeded("account", Id(101), "Alpha")) + .WithAttributes("account", "statecode", "statuscode"); + var shuffler = NewShuffler(); + shuffler.TestDeferState("account", Id(1), Id(101), 1, 2); + shuffler.TestDeferState("account", Id(2), Guid.Empty, 1, 2); + + shuffler.TestFlushDeferredStateChanges(); + + Assert.That(Service.CountOf("SetState"), Is.EqualTo(1), DumpAll()); + Assert.That( + Org.Logger.Logged("Skipping 1 deferred state change(s) for records that were not written"), + DumpAll()); + Assert.That(Org.Logger.Logged("Deferred state changes: 1 applied, 0 failed, 1 skipped"), DumpAll()); + } + + /// + /// savedquery and duplicaterule do not take a plain state update, so they never go + /// through UpdateMultiple even where the platform offers it. A published savedquery is + /// also the one place where the state written differs from the state logged. + /// + [Test] + public void savedquery_never_goes_through_UpdateMultiple_even_where_it_is_supported() + { + Online() + .WithEntity(Seeded("savedquery", Id(101), "Active accounts")) + .WithAttributes("savedquery", "statecode", "statuscode"); + var shuffler = NewShuffler(); + shuffler.TestDeferState("savedquery", Id(1), Id(101), 1, 1, 1, "Active accounts"); + + shuffler.TestFlushDeferredStateChanges(); + + Assert.That(Service.CountOf("UpdateMultiple"), Is.EqualTo(0), DumpAll()); + Assert.That(Service.CountOf("SetState"), Is.EqualTo(1), DumpAll()); + var setState = Service.RequestsNamed("SetState")[0]; + Assert.That(Option(setState, "State").Value, Is.EqualTo(1), DumpAll()); + Assert.That(Option(setState, "Status").Value, Is.EqualTo(2), "1/1 is rewritten to 1/2 for savedquery"); + Assert.That( + Org.Logger.Logged("001 SetState (deferred): Active accounts: 1/1"), + "the line reports the requested state, not the one sent"); + } + + // ---- the owner pass ------------------------------------------------- + + /// + /// There is no bulk assign, so the owner pass has no fallback chain to choose from. + /// + [Test] + public void Deferred_owners_are_assigned_one_at_a_time() + { + Online() + .WithEntity(Seeded("account", Id(101), "Alpha"), Seeded("account", Id(102), "Beta")) + .WithAttributes("account", "ownerid"); + var shuffler = NewShuffler(); + shuffler.TestDeferOwner("account", Id(1), Id(101), Owner(9), 1, "Alpha"); + shuffler.TestDeferOwner("account", Id(2), Id(102), Owner(9), 2, "Beta"); + + shuffler.TestFlushDeferredOwnerChanges(); + + Assert.That(Service.CountOf("Assign"), Is.EqualTo(2), DumpAll()); + Assert.That(Service.CountOf("UpdateMultiple"), Is.EqualTo(0), DumpAll()); + Assert.That(Org.Logger.Logged("001 Assigned (deferred): Alpha to systemuser"), DumpAll()); + Assert.That(Org.Logger.Logged("Deferred owner changes: 2 applied, 0 failed, 0 skipped"), DumpAll()); + } + + [Test] + public void Owner_changes_for_unwritten_records_are_skipped() + { + OnPrem() + .WithEntity(Seeded("account", Id(101), "Alpha")) + .WithAttributes("account", "ownerid"); + var shuffler = NewShuffler(); + shuffler.TestDeferOwner("account", Id(1), Id(101), Owner(9)); + shuffler.TestDeferOwner("account", Id(2), Guid.Empty, Owner(9)); + + shuffler.TestFlushDeferredOwnerChanges(); + + Assert.That(Service.CountOf("Assign"), Is.EqualTo(1), DumpAll()); + Assert.That( + Org.Logger.Logged("Skipping 1 deferred owner change(s) for records that were not written"), + DumpAll()); + Assert.That(Org.Logger.Logged("Deferred owner changes: 1 applied, 0 failed, 1 skipped"), DumpAll()); + } + + /// + /// KNOWN DEFECT - asserts what the code does today, not what it should do. + /// + /// + /// + /// The owner pass reaches AssignRequest through the fluent helper + /// container.Principal(x).On(y).Assign(), and OperationsSet2.Assign catches every + /// exception and returns false. The catch in FlushDeferredOwnerChanges is therefore dead + /// code: a rejected assign increments applied, logs "Assigned (deferred)", and StopOnError + /// never fires. The records keep the owner the import was supposed to change. + /// + /// + /// A fix would read the bool the helper returns, or send the AssignRequest directly. When + /// that lands this test fails, which is the point of writing it down. + /// + /// + [Test] + public void A_failed_assign_is_silently_counted_as_applied() + { + OnPrem() + .WithEntity(Seeded("account", Id(101), "Alpha")) + .WithAttributes("account", "ownerid"); + Service.ThrowAlways( + "Assign", + new FaultException( + new OrganizationServiceFault(), "principal has no access")); + var shuffler = NewShuffler(stopOnError: true); + shuffler.TestDeferOwner("account", Id(1), Id(101), Owner(9), 1, "Alpha"); + + Assert.DoesNotThrow(() => shuffler.TestFlushDeferredOwnerChanges(), DumpAll()); + + Assert.That(Org.Logger.Logged("001 Assigned (deferred): Alpha to systemuser"), DumpAll()); + Assert.That(Org.Logger.Logged("Deferred owner changes: 1 applied, 0 failed, 0 skipped"), DumpAll()); + Assert.That(Org.Logger.Logged("Assign Failed (deferred)"), Is.False, DumpAll()); + } + + /// + /// KNOWN DEFECT - asserts what the code does today, not what it should do. + /// + /// + /// The fluent helper reads Principal as the assignee and On as the target, but the owner + /// pass calls container.Principal(record).On(owner), so the request that goes out + /// asks to assign the account to itself with the user as the target. The swallowed + /// exception above is what keeps this invisible. Fixing either one of these two defects + /// without the other only changes which of them is reported. + /// + [Test] + public void The_deferred_assign_sends_the_record_and_the_owner_the_wrong_way_round() + { + OnPrem() + .WithEntity(Seeded("account", Id(101), "Alpha")) + .WithAttributes("account", "ownerid"); + var shuffler = NewShuffler(); + shuffler.TestDeferOwner("account", Id(1), Id(101), Owner(9), 1, "Alpha"); + + shuffler.TestFlushDeferredOwnerChanges(); + + var assign = Service.RequestsNamed("Assign")[0]; + var assignee = (EntityReference)assign["Assignee"]; + var target = (EntityReference)assign["Target"]; + Assert.That(assignee.LogicalName, Is.EqualTo("account"), "should be the systemuser"); + Assert.That(target.LogicalName, Is.EqualTo("systemuser"), "should be the account"); + } + + // ---- the option seen from a whole block ----------------------------- + + private static Types.DataBlock Block(bool defer) + { + var builder = DefinitionXml.DataBlock("Accounts", "account").BatchSize(10); + if (defer) + { + builder = builder.DeferStateAndOwner(); + } + return builder.DeserializeBlock(); + } + + /// + /// A block carrying nothing but state and owner is already a second pass of its own - + /// the common shape where state changes live in a separate UpdateOnly block. Deferring + /// there would strip every record down to nothing, so the option is refused rather than + /// obeyed. + /// + [Test] + public void A_state_and_owner_only_block_ignores_the_option() + { + Online().WithMetadata("account").WithAttributes("account", "statecode", "statuscode"); + var sources = new EntityCollection { EntityName = "account" }; + sources.Entities.Add(Stateful("account", Id(1), 1, 2)); + sources.Entities.Add(Stateful("account", Id(2), 1, 2)); + sources.Entities[0].Attributes.Remove("name"); + sources.Entities[1].Attributes.Remove("name"); + + NewShuffler().TestImportDataBlock(Block(defer: true), sources); + + Assert.That( + Org.Logger.Logged( + "DeferStateAndOwner ignored - this block carries no attributes besides state and owner"), + DumpAll()); + Assert.That(Service.CountOf("CreateMultiple"), Is.EqualTo(0), DumpAll()); + } + + /// + /// The end-to-end claim: a block whose records set an owner batches with the option on + /// and does not without it. + /// + [Test] + public void Deferring_lets_a_block_that_sets_owner_batch() + { + Online().WithMetadata("account").WithAttributes("account", "ownerid"); + var outcome = NewShuffler().TestImportDataBlock(Block(defer: true), Owned()); + + Assert.That(Org.Logger.Logged("DeferStateAndOwner enabled - state/owner will be applied in second pass"), DumpAll()); + Assert.That(outcome.Created, Is.EqualTo(2), DumpAll()); + Assert.That(Service.CountOf("CreateMultiple"), Is.EqualTo(1), DumpAll()); + Assert.That(Service.CountOf("Assign"), Is.EqualTo(2), DumpAll()); + Assert.That(Org.Logger.Logged("Deferred owner changes: 2 applied, 0 failed, 0 skipped"), DumpAll()); + } + + [Test] + public void Without_the_option_the_same_block_is_written_one_record_at_a_time() + { + Online().WithMetadata("account").WithAttributes("account", "ownerid"); + var outcome = NewShuffler().TestImportDataBlock(Block(defer: false), Owned()); + + Assert.That(outcome.Created, Is.EqualTo(2), DumpAll()); + Assert.That(Service.CountOf("CreateMultiple"), Is.EqualTo(0), DumpAll()); + Assert.That(Service.Created.Count, Is.EqualTo(2), DumpAll()); + Assert.That(Service.CountOf("Assign"), Is.EqualTo(0), DumpAll()); + } + + private static EntityCollection Owned() + { + var sources = new EntityCollection { EntityName = "account" }; + for (var seed = 1; seed <= 2; seed++) + { + var record = Record("account", Id(seed), "Account " + seed); + record["ownerid"] = Owner(9); + sources.Entities.Add(record); + } + return sources; + } + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Layer3/GuidRemappingTests.cs b/tests/Xrm.Shuffle.Core.Tests/Layer3/GuidRemappingTests.cs new file mode 100644 index 0000000..33d2a46 --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Layer3/GuidRemappingTests.cs @@ -0,0 +1,289 @@ +namespace Cinteros.Crm.Utils.Shuffle.Tests.Layer3 +{ + using System; + using Cinteros.Crm.Utils.Shuffle.Tests.Helpers; + using Microsoft.Xrm.Sdk; + using NUnit.Framework; + + /// + /// The guid map, and what gets rewritten through it. + /// + /// + /// + /// A definition carries the ids of the source system. The target assigns its own on create, + /// so every lookup written after that has to be translated. The map is filled as records are + /// written and read by ReplaceGuids on each record before it is saved, which is why block + /// order in a definition is load-bearing: a block pointing at another block only works if + /// that other block ran first. + /// + /// + /// MapGuid is deliberately picky - it declines an empty id on either side, an id that did + /// not change, and an id it has already mapped. RecordCreatedId wraps it precisely because + /// the deferred state and owner queues need the real id in the two cases the map declines. + /// + /// + [TestFixture] + public class GuidRemappingTests : FakeOrgTestBase + { + private static Entity Lookup(string entityLogicalName, Guid id, string attribute, Guid target) + { + var entity = Record(entityLogicalName, id); + entity[attribute] = new EntityReference("account", target); + return entity; + } + + private static Types.DataBlock Creates(string blockName, string entityLogicalName) + { + return DefinitionXml.DataBlock(blockName, entityLogicalName) + .BatchSize(10) + .DeserializeBlock(); + } + + // --- MapGuid: what it accepts ------------------------------------------------------- + + [Test] + public void A_changed_id_is_mapped() + { + OnPrem(); + var shuffler = NewShuffler(); + + shuffler.TestMapGuid(Id(1), Id(101)); + + Assert.That(shuffler.TestGuidMap.Count, Is.EqualTo(1)); + Assert.That(shuffler.TestGuidMap[Id(1)], Is.EqualTo(Id(101))); + } + + [Test] + public void An_empty_source_id_is_not_mapped() + { + OnPrem(); + var shuffler = NewShuffler(); + + shuffler.TestMapGuid(Guid.Empty, Id(101)); + + Assert.That(shuffler.TestGuidMap, Is.Empty); + } + + [Test] + public void An_empty_target_id_is_not_mapped() + { + OnPrem(); + var shuffler = NewShuffler(); + + shuffler.TestMapGuid(Id(1), Guid.Empty); + + Assert.That(shuffler.TestGuidMap, Is.Empty); + } + + /// + /// An unchanged id needs no translation, so mapping it would only cost a lookup. + /// + [Test] + public void An_unchanged_id_is_not_mapped() + { + OnPrem(); + var shuffler = NewShuffler(); + + shuffler.TestMapGuid(Id(1), Id(1)); + + Assert.That(shuffler.TestGuidMap, Is.Empty); + } + + /// + /// First writer wins. Two source records sharing an id is a broken definition, and + /// silently retargeting every later lookup would be worse than keeping the first answer. + /// + [Test] + public void An_already_mapped_id_keeps_its_first_target() + { + OnPrem(); + var shuffler = NewShuffler(); + + shuffler.TestMapGuid(Id(1), Id(101)); + shuffler.TestMapGuid(Id(1), Id(102)); + + Assert.That(shuffler.TestGuidMap[Id(1)], Is.EqualTo(Id(101))); + } + + // --- RecordCreatedId: the two cases the map declines -------------------------------- + + [Test] + public void RecordCreatedId_fills_a_deferred_id_even_when_MapGuid_declines_an_unchanged_id() + { + OnPrem(); + var shuffler = NewShuffler(); + shuffler.TestDeferState("account", Id(1), Guid.Empty, 1, 2); + + shuffler.TestRecordCreatedId(Id(1), Id(1)); + + Assert.That(shuffler.TestGuidMap, Is.Empty, "an unchanged id is still not mapped"); + Assert.That(shuffler.TestDeferredActualId(Id(1)), Is.EqualTo(Id(1))); + } + + [Test] + public void RecordCreatedId_fills_a_deferred_id_even_when_the_source_id_is_already_mapped() + { + OnPrem(); + var shuffler = NewShuffler(); + shuffler.TestMapGuid(Id(1), Id(101)); + shuffler.TestDeferOwner("account", Id(1), Guid.Empty, new EntityReference("systemuser", Id(9))); + + shuffler.TestRecordCreatedId(Id(1), Id(102)); + + Assert.That(shuffler.TestGuidMap[Id(1)], Is.EqualTo(Id(101)), "the map keeps the first target"); + Assert.That(shuffler.TestDeferredOwnerActualId(Id(1)), Is.EqualTo(Id(102))); + } + + [Test] + public void RecordCreatedId_leaves_the_deferred_queues_alone_for_an_empty_id() + { + OnPrem(); + var shuffler = NewShuffler(); + shuffler.TestDeferState("account", Id(1), Guid.Empty, 1, 2); + + shuffler.TestRecordCreatedId(Id(1), Guid.Empty); + + Assert.That(shuffler.TestDeferredActualId(Id(1)), Is.EqualTo(Guid.Empty)); + } + + // --- ReplaceGuids ------------------------------------------------------------------ + + [Test] + public void A_mapped_lookup_is_rewritten_in_place() + { + OnPrem(); + var shuffler = NewShuffler(); + shuffler.TestMapGuid(Id(1), Id(101)); + var record = Lookup("contact", Id(5), "parentcustomerid", Id(1)); + + shuffler.TestReplaceGuids(record); + + Assert.That(((EntityReference)record["parentcustomerid"]).Id, Is.EqualTo(Id(101))); + } + + [Test] + public void An_unmapped_lookup_is_left_alone() + { + OnPrem(); + var shuffler = NewShuffler(); + shuffler.TestMapGuid(Id(1), Id(101)); + var record = Lookup("contact", Id(5), "parentcustomerid", Id(2)); + + shuffler.TestReplaceGuids(record); + + Assert.That(((EntityReference)record["parentcustomerid"]).Id, Is.EqualTo(Id(2))); + } + + [Test] + public void Every_mapped_lookup_on_a_record_is_rewritten() + { + OnPrem(); + var shuffler = NewShuffler(); + shuffler.TestMapGuid(Id(1), Id(101)); + shuffler.TestMapGuid(Id(2), Id(102)); + var record = Record("contact", Id(5)); + record["parentcustomerid"] = new EntityReference("account", Id(1)); + record["originatingleadid"] = new EntityReference("account", Id(2)); + + shuffler.TestReplaceGuids(record); + + Assert.That(((EntityReference)record["parentcustomerid"]).Id, Is.EqualTo(Id(101))); + Assert.That(((EntityReference)record["originatingleadid"]).Id, Is.EqualTo(Id(102))); + } + + /// + /// A raw Guid attribute that happens to be mapped is not a lookup - it is a + /// uniqueidentifier column holding an id the target does not know about. + /// + [Test] + public void A_mapped_raw_guid_attribute_is_only_noted_when_ids_are_not_carried_over() + { + OnPrem(); + var shuffler = NewShuffler(); + shuffler.TestMapGuid(Id(1), Id(101)); + var record = Record("contact", Id(5)); + record["cint_sourceid"] = Id(1); + + shuffler.TestReplaceGuids(record, includeId: false); + + Assert.That((Guid)record["cint_sourceid"], Is.EqualTo(Id(1)), "left as the source value"); + Assert.That(Org.Logger.Logged("care about the guid of the object"), DumpAll()); + } + + /// + /// With CreateWithId the target keeps the source ids, so a mapped raw guid means the + /// definition asked for something the import cannot honour - and it says so rather than + /// writing a value it knows is wrong. + /// + [Test] + public void A_mapped_raw_guid_attribute_is_refused_when_ids_are_carried_over() + { + OnPrem(); + var shuffler = NewShuffler(); + shuffler.TestMapGuid(Id(1), Id(101)); + var record = Record("contact", Id(5)); + record["cint_sourceid"] = Id(1); + + Assert.That( + () => shuffler.TestReplaceGuids(record, includeId: true), + Throws.TypeOf()); + } + + // --- across blocks ----------------------------------------------------------------- + + /// + /// The point of all of it: a later block lands its lookups on the records the earlier + /// block actually wrote, not on the ids the source system used. + /// + [Test] + public void A_later_block_points_at_what_the_earlier_block_actually_wrote() + { + Online().WithMetadata("account").WithMetadata("contact"); + var shuffler = NewShuffler(); + var accounts = new EntityCollection { EntityName = "account" }; + accounts.Entities.Add(Record("account", Id(1), "Alpha")); + accounts.Entities.Add(Record("account", Id(2), "Beta")); + var contacts = new EntityCollection { EntityName = "contact" }; + contacts.Entities.Add(Lookup("contact", Id(11), "parentcustomerid", Id(1))); + contacts.Entities.Add(Lookup("contact", Id(12), "parentcustomerid", Id(2))); + + var accountOutcome = shuffler.TestImportDataBlock(Creates("Accounts", "account"), accounts); + var contactOutcome = shuffler.TestImportDataBlock(Creates("Contacts", "contact"), contacts); + + Assert.That(accountOutcome.Created, Is.EqualTo(2), DumpAll()); + Assert.That(contactOutcome.Created, Is.EqualTo(2), DumpAll()); + var written = RecordingOrganizationService.TargetsOf( + Service.RequestsNamed("CreateMultiple")[0]); + var pointed = RecordingOrganizationService.TargetsOf( + Service.RequestsNamed("CreateMultiple")[1]); + Assert.That( + ((EntityReference)pointed[0]["parentcustomerid"]).Id, + Is.EqualTo(written[0].Id), + DumpAll()); + Assert.That( + ((EntityReference)pointed[1]["parentcustomerid"]).Id, + Is.EqualTo(written[1].Id), + DumpAll()); + Assert.That( + ((EntityReference)pointed[0]["parentcustomerid"]).Id, + Is.Not.EqualTo(Id(1)), + "the source id should not have survived"); + } + + [Test] + public void Creating_a_block_of_records_maps_every_id_it_assigned() + { + Online().WithMetadata("account"); + var shuffler = NewShuffler(); + var accounts = new EntityCollection { EntityName = "account" }; + accounts.Entities.Add(Record("account", Id(1), "Alpha")); + accounts.Entities.Add(Record("account", Id(2), "Beta")); + + shuffler.TestImportDataBlock(Creates("Accounts", "account"), accounts); + + Assert.That(shuffler.TestGuidMap.Count, Is.EqualTo(2), DumpAll()); + Assert.That(shuffler.TestGuidMap.ContainsKey(Id(1)), DumpAll()); + Assert.That(shuffler.TestGuidMap.ContainsKey(Id(2)), DumpAll()); + } + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Layer3/PendingCreateReferenceTests.cs b/tests/Xrm.Shuffle.Core.Tests/Layer3/PendingCreateReferenceTests.cs new file mode 100644 index 0000000..e6881a0 --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Layer3/PendingCreateReferenceTests.cs @@ -0,0 +1,275 @@ +namespace Cinteros.Crm.Utils.Shuffle.Tests.Layer3 +{ + using System; + using Cinteros.Crm.Utils.Shuffle.Tests.Helpers; + using Microsoft.Xrm.Sdk; + using NUnit.Framework; + + /// + /// The guard that keeps batching from writing a lookup to a record that does not exist yet. + /// + /// + /// + /// Batching defers the create, so the id the target assigns is not known until the batch is + /// flushed. A later record in the same block pointing at an earlier one would therefore be + /// written with the id of the source system - a lookup into nothing. + /// + /// + /// ReferencesPendingCreate is checked before each record is prepared, and a hit flushes the + /// create batch early so the ids exist and the guid map can translate them. The cost is a + /// shorter batch; the alternative is silent data loss, so the guard is deliberately eager - + /// it looks at every attribute of the record, matches on the id of the source system rather + /// than on what the record holds now, and does not care which entity is referenced. + /// + /// + [TestFixture] + public class PendingCreateReferenceTests : FakeOrgTestBase + { + private static Shuffler.TestCreateBatch Pending(params Guid[] oldIds) + { + var batch = new Shuffler.TestCreateBatch(); + var seed = 500; + foreach (var oldId in oldIds) + { + batch.Add(Record("account", Id(seed++)), oldId); + } + return batch; + } + + private static Entity Pointing(Guid id, string attribute, object value) + { + var entity = Record("contact", id); + entity[attribute] = value; + return entity; + } + + private static Types.DataBlock Creates() + { + return DefinitionXml.DataBlock("Accounts", "account") + .BatchSize(10) + .DeserializeBlock(); + } + + private static EntityCollection Referencing() + { + var sources = new EntityCollection { EntityName = "account" }; + sources.Entities.Add(Record("account", Id(1), "Alpha")); + sources.Entities.Add(Record("account", Id(2), "Beta")); + sources.Entities.Add(Child(Id(3), "Gamma", Id(1))); + sources.Entities.Add(Record("account", Id(4), "Delta")); + return sources; + } + + private static Entity Child(Guid id, string name, Guid parent) + { + var entity = Record("account", id, name); + entity["parentaccountid"] = new EntityReference("account", parent); + return entity; + } + + // --- the guard itself --------------------------------------------------------------- + + [Test] + public void An_empty_batch_is_never_referenced() + { + OnPrem(); + + Assert.That( + Shuffler.TestReferencesPendingCreate( + Pointing(Id(5), "parentcustomerid", new EntityReference("account", Id(1))), + Pending()), + Is.False); + } + + [Test] + public void A_lookup_to_a_queued_record_is_a_hit() + { + OnPrem(); + + Assert.That( + Shuffler.TestReferencesPendingCreate( + Pointing(Id(5), "parentcustomerid", new EntityReference("account", Id(1))), + Pending(Id(1)))); + } + + /// + /// A uniqueidentifier column counts too - the guard cannot tell a lookup stored as a raw + /// guid from one stored as an EntityReference, and guessing wrong loses data. + /// + [Test] + public void A_raw_guid_attribute_pointing_at_a_queued_record_is_a_hit() + { + OnPrem(); + + Assert.That( + Shuffler.TestReferencesPendingCreate( + Pointing(Id(5), "cint_sourceid", Id(1)), + Pending(Id(1)))); + } + + [Test] + public void A_lookup_to_something_not_queued_is_not_a_hit() + { + OnPrem(); + + Assert.That( + Shuffler.TestReferencesPendingCreate( + Pointing(Id(5), "parentcustomerid", new EntityReference("account", Id(2))), + Pending(Id(1))), + Is.False); + } + + [Test] + public void An_empty_lookup_is_not_a_hit() + { + OnPrem(); + + Assert.That( + Shuffler.TestReferencesPendingCreate( + Pointing(Id(5), "parentcustomerid", new EntityReference("account", Guid.Empty)), + Pending(Id(1))), + Is.False); + } + + [Test] + public void A_record_with_no_lookups_at_all_is_not_a_hit() + { + OnPrem(); + + Assert.That( + Shuffler.TestReferencesPendingCreate(Record("contact", Id(5)), Pending(Id(1))), + Is.False); + } + + [Test] + public void Any_one_of_several_queued_records_is_enough() + { + OnPrem(); + + Assert.That( + Shuffler.TestReferencesPendingCreate( + Pointing(Id(5), "parentcustomerid", new EntityReference("account", Id(3))), + Pending(Id(1), Id(2), Id(3)))); + } + + /// + /// The match is on the id of the source system, not on the id the queued entity holds. + /// The no-match create branch blanks the id before queueing, so by the time the guard + /// runs the entity in the batch usually has no id at all. + /// + [Test] + public void The_match_is_on_the_source_id_not_on_what_the_queued_entity_holds() + { + OnPrem(); + var batch = new Shuffler.TestCreateBatch(); + var queued = Record("account", Guid.Empty); + batch.Add(queued, Id(1)); + + Assert.That( + Shuffler.TestReferencesPendingCreate( + Pointing(Id(5), "parentcustomerid", new EntityReference("account", Id(1))), + batch)); + Assert.That(queued.Id, Is.EqualTo(Guid.Empty), "the queued record still has no id"); + } + + // --- what the guard does to a block ------------------------------------------------ + + /// + /// The third record points at the first, so the batch is flushed before it is prepared, + /// and the block ends up sending two batches where an unreferenced block sends one. + /// + [Test] + public void A_record_pointing_at_an_earlier_one_cuts_the_batch_short() + { + Online().WithMetadata("account"); + + var outcome = NewShuffler().TestImportDataBlock(Creates(), Referencing()); + + Assert.That(outcome.Created, Is.EqualTo(4), DumpAll()); + Assert.That(Service.CountOf("CreateMultiple"), Is.EqualTo(2), DumpAll()); + Assert.That( + RecordingOrganizationService.TargetsOf( + Service.RequestsNamed("CreateMultiple")[0]).Count, + Is.EqualTo(2), + "cut short at the two records queued before the reference"); + } + + [Test] + public void A_block_with_no_cross_references_stays_one_batch() + { + Online().WithMetadata("account"); + var sources = new EntityCollection { EntityName = "account" }; + sources.Entities.Add(Record("account", Id(1), "Alpha")); + sources.Entities.Add(Record("account", Id(2), "Beta")); + sources.Entities.Add(Record("account", Id(3), "Gamma")); + sources.Entities.Add(Record("account", Id(4), "Delta")); + + var outcome = NewShuffler().TestImportDataBlock(Creates(), sources); + + Assert.That(outcome.Created, Is.EqualTo(4), DumpAll()); + Assert.That(Service.CountOf("CreateMultiple"), Is.EqualTo(1), DumpAll()); + } + + /// + /// The point of flushing early: the lookup is rewritten to the id the target actually + /// assigned, not the id the source system used. + /// + [Test] + public void The_lookup_is_written_against_the_id_the_target_assigned() + { + Online().WithMetadata("account"); + + NewShuffler().TestImportDataBlock(Creates(), Referencing()); + + var first = RecordingOrganizationService.TargetsOf( + Service.RequestsNamed("CreateMultiple")[0]); + var second = RecordingOrganizationService.TargetsOf( + Service.RequestsNamed("CreateMultiple")[1]); + var written = ((EntityReference)second[0]["parentaccountid"]).Id; + Assert.That(written, Is.EqualTo(first[0].Id), DumpAll()); + Assert.That(written, Is.Not.EqualTo(Id(1)), "the source id should not have survived"); + } + + /// + /// A record pointing at one that an earlier batch already wrote does not cut the batch + /// again - the pending list no longer holds it, so the guard has nothing to hit and the + /// guid map is what supplies the id. + /// + [Test] + public void Pointing_at_an_already_written_record_does_not_cut_the_batch() + { + Online().WithMetadata("account"); + var sources = Referencing(); + sources.Entities[3]["parentaccountid"] = new EntityReference("account", Id(1)); + + var outcome = NewShuffler().TestImportDataBlock(Creates(), sources); + + Assert.That(outcome.Created, Is.EqualTo(4), DumpAll()); + Assert.That(Service.CountOf("CreateMultiple"), Is.EqualTo(2), DumpAll()); + var second = RecordingOrganizationService.TargetsOf( + Service.RequestsNamed("CreateMultiple")[1]); + Assert.That(second.Count, Is.EqualTo(2), "both referencing records batch together"); + } + + /// + /// The early flush takes whatever is queued, so a reference to the record immediately + /// before it leaves a batch of one - which the dispatcher then sends as a plain Create. + /// The guard trades throughput for correctness and does not try to soften that. + /// + [Test] + public void An_early_flush_of_one_queued_record_goes_out_as_a_plain_create() + { + Online().WithMetadata("account"); + var sources = new EntityCollection { EntityName = "account" }; + sources.Entities.Add(Record("account", Id(1), "Alpha")); + sources.Entities.Add(Child(Id(2), "Beta", Id(1))); + sources.Entities.Add(Record("account", Id(3), "Gamma")); + + var outcome = NewShuffler().TestImportDataBlock(Creates(), sources); + + Assert.That(outcome.Created, Is.EqualTo(3), DumpAll()); + Assert.That(Service.CountOf("Create"), Is.EqualTo(1), DumpAll()); + Assert.That(Service.CountOf("CreateMultiple"), Is.EqualTo(1), DumpAll()); + } + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Properties/AssemblyInfo.cs b/tests/Xrm.Shuffle.Core.Tests/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..b104ded --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Properties/AssemblyInfo.cs @@ -0,0 +1,17 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +[assembly: AssemblyCompany("Jonas, Imran and the Power Platform Community")] +[assembly: AssemblyProduct("Shuffle Tools for XrmToolBox")] +[assembly: AssemblyTitle("Xrm.Shuffle.Core.Tests")] +[assembly: AssemblyDescription("Unit tests for the Shuffle core")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCopyright("")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +[assembly: ComVisible(false)] +[assembly: Guid("7c3e9a21-4b8d-4e52-9f1c-6d0a5b2e4417")] + +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/tests/Xrm.Shuffle.Core.Tests/Regressions/BatchErrorReportingTests.cs b/tests/Xrm.Shuffle.Core.Tests/Regressions/BatchErrorReportingTests.cs new file mode 100644 index 0000000..f28ca9d --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Regressions/BatchErrorReportingTests.cs @@ -0,0 +1,253 @@ +namespace Cinteros.Crm.Utils.Shuffle.Tests.Regressions +{ + using System.ServiceModel; + using Cinteros.Crm.Utils.Shuffle; + using Cinteros.Crm.Utils.Shuffle.Tests.Helpers; + using Microsoft.Xrm.Sdk; + using NUnit.Framework; + + /// + /// Which record a batch failure is blamed on, and the lowest rung of the upsert chain. + /// + /// + /// + /// Without batching a failure is reported by the record loop, which knows exactly which + /// record it was working on. A batch breaks that: the loop is several records past the one + /// that faulted by the time the platform answers, so the catch would name whichever record + /// happened to fill the batch. StopOnBatchError exists to carry the right label out of the + /// flush and into that catch. + /// + /// + /// The other half of this fixture is FlushUpsertsAsCreateUpdate, the rung reached when the + /// target has no Upsert message at all. It has to work out create-versus-update from the + /// fault a create came back with, which is guesswork against error codes and message text. + /// + /// + [TestFixture] + public class BatchErrorReportingTests : ShuffleTestBase + { + private const int DuplicateRecordEntityKey = -2147220937; + private const int DuplicateRecord = -2147220685; + private const int SomethingElse = -2147220989; + + private static Shuffler.TestUpsertBatch Upserts(int count) + { + var batch = new Shuffler.TestUpsertBatch(); + for (var i = 1; i <= count; i++) + { + batch.Add(Record("account", Id(i), "Account " + i), Id(i), "account Acme " + i); + } + return batch; + } + + private static FaultException Fault(string message, int errorCode) + { + return new FaultException( + new OrganizationServiceFault { Message = message, ErrorCode = errorCode }, + message); + } + + #region The label a batch failure is reported under + + [Test] + public void The_label_is_the_position_and_the_identifier_of_the_record_that_failed() + { + var shuffler = NewShuffler(stopOnError: true); + + Assert.That(shuffler.TestStopOnBatchError(7, "Acme Corp"), Is.True); + Assert.That(shuffler.TestBatchFailureLabel, Is.EqualTo("007 Acme Corp")); + } + + /// + /// The position is padded the same way the log lines are, so the label reads like the + /// line the record would have produced had it succeeded. + /// + [Test] + public void The_position_is_padded_to_three_digits_but_not_truncated() + { + var shuffler = NewShuffler(stopOnError: true); + + shuffler.TestStopOnBatchError(3, "early"); + Assert.That(shuffler.TestBatchFailureLabel, Is.EqualTo("003 early")); + + shuffler.TestStopOnBatchError(1234, "late"); + Assert.That(shuffler.TestBatchFailureLabel, Is.EqualTo("1234 late")); + } + + /// + /// Without StopOnError there is nothing to report: the flush carries on and each failed + /// record has already been logged under its own number. + /// + [Test] + public void Without_StopOnError_nothing_is_labelled_and_the_caller_is_told_to_continue() + { + var shuffler = NewShuffler(); + + Assert.That(shuffler.TestStopOnBatchError(7, "Acme Corp"), Is.False); + Assert.That(shuffler.TestBatchFailureLabel, Is.Null); + } + + #endregion The label a batch failure is reported under + + #region The lowest upsert rung: create, then update if it was already there + + [Test] + public void Every_record_is_created_when_none_of_them_exists() + { + Service.OnCreate(entity => Id(500)); + + var outcome = NewShuffler().TestFlushUpsertsAsCreateUpdate(Upserts(2)); + + Assert.That(outcome.Created, Is.EqualTo(2), outcome.ToString()); + Assert.That(outcome.Updated, Is.EqualTo(0), outcome.ToString()); + Recorder.AssertLogged("Falling back to Create/Update for 2 records (Upsert not available)"); + Recorder.AssertSent("001 Created: account Acme 1"); + } + + [Test] + public void A_duplicate_key_fault_is_read_as_already_there_and_becomes_an_update() + { + Service.OnCreate(entity => + { + throw Fault("A record with these values exists", DuplicateRecordEntityKey); + }); + + var outcome = NewShuffler().TestFlushUpsertsAsCreateUpdate(Upserts(2)); + + Assert.That(outcome.Updated, Is.EqualTo(2), outcome.ToString()); + Assert.That(outcome.Created, Is.EqualTo(0), outcome.ToString()); + Assert.That(Service.CountOf("Update"), Is.EqualTo(2), DumpAll()); + Recorder.AssertSent("001 Updated: account Acme 1"); + } + + [Test] + public void The_other_duplicate_error_code_is_read_the_same_way() + { + Service.OnCreate(entity => + { + throw Fault("Duplicate detected by a rule", DuplicateRecord); + }); + + var outcome = NewShuffler().TestFlushUpsertsAsCreateUpdate(Upserts(2)); + + Assert.That(outcome.Updated, Is.EqualTo(2), outcome.ToString()); + } + + /// + /// A fault with no recognised error code still counts as already there when its message + /// says so, which is what keeps the rung working against targets that report duplicates + /// through text rather than a code. + /// + [Test] + public void A_fault_whose_message_says_it_already_exists_is_also_an_update() + { + Service.OnCreate(entity => + { + throw Fault("The record already exists", SomethingElse); + }); + + var outcome = NewShuffler().TestFlushUpsertsAsCreateUpdate(Upserts(2)); + + Assert.That(outcome.Updated, Is.EqualTo(2), outcome.ToString()); + } + + [Test] + public void The_word_duplicate_in_lower_case_is_enough_on_its_own() + { + Service.OnCreate(entity => + { + throw Fault("rejected as a duplicate of another record", SomethingElse); + }); + + var outcome = NewShuffler().TestFlushUpsertsAsCreateUpdate(Upserts(2)); + + Assert.That(outcome.Updated, Is.EqualTo(2), outcome.ToString()); + } + + /// + /// Known defect. The message check is case sensitive, so a target that capitalises + /// Duplicate falls past it and the record is reported as a create failure even though + /// it exists and an update would have worked. + /// + /// + /// The error codes cover the platform messages, so this is only reached by a target or + /// a plugin that raises its own text. Fixing it means comparing case insensitively, at + /// which point this test should fail and be rewritten as an update. + /// + [Test] + public void Known_defect_a_capitalised_Duplicate_message_is_not_recognised() + { + Service.OnCreate(entity => + { + throw Fault("Duplicate record found", SomethingElse); + }); + + var outcome = NewShuffler().TestFlushUpsertsAsCreateUpdate(Upserts(2)); + + Assert.That(outcome.Failed, Is.EqualTo(2), outcome.ToString()); + Assert.That(outcome.Updated, Is.EqualTo(0), "an update was never attempted"); + Assert.That(Service.CountOf("Update"), Is.EqualTo(0), DumpAll()); + Recorder.AssertSent("001 Create Failed: account Acme 1 Duplicate record found"); + } + + [Test] + public void A_create_that_fails_for_any_other_reason_is_a_failure() + { + Service.OnCreate(entity => + { + throw Fault("Privilege denied", SomethingElse); + }); + + var outcome = NewShuffler().TestFlushUpsertsAsCreateUpdate(Upserts(2)); + + Assert.That(outcome.Failed, Is.EqualTo(2), outcome.ToString()); + Assert.That(Service.CountOf("Update"), Is.EqualTo(0), DumpAll()); + } + + /// + /// The record exists but the update fails too - reported under its own message so the + /// log says which of the two calls went wrong. + /// + [Test] + public void An_update_that_fails_after_a_duplicate_create_is_reported_as_an_update_failure() + { + Service.OnCreate(entity => + { + throw Fault("duplicate", DuplicateRecord); + }); + Service.OnUpdate(entity => + { + throw Fault("Read only field", SomethingElse); + }); + + var outcome = NewShuffler().TestFlushUpsertsAsCreateUpdate(Upserts(2)); + + Assert.That(outcome.Failed, Is.EqualTo(2), outcome.ToString()); + Assert.That(outcome.Updated, Is.EqualTo(0), outcome.ToString()); + Recorder.AssertSent("001 Update Failed (fallback): account Acme 1 Read only field"); + } + + /// + /// StopOnError stops at the first record and says how many of the batch never ran, so + /// the run can be resumed without guessing where it stopped. + /// + [Test] + public void StopOnError_abandons_the_rest_of_the_batch_and_says_how_many_were_left() + { + Service.OnCreate(entity => + { + throw Fault("Privilege denied", SomethingElse); + }); + var shuffler = NewShuffler(stopOnError: true); + + Assert.Throws>( + () => shuffler.TestFlushUpsertsAsCreateUpdate(Upserts(3))); + + Assert.That(shuffler.TestBatchFailureLabel, Is.EqualTo("001 account Acme 1")); + Recorder.AssertLogged("StopOnError: aborting, 2 record(s) in this batch were not executed"); + Assert.That(Service.CountOf("Create"), Is.EqualTo(1), "the two records behind it never ran"); + } + + #endregion The lowest upsert rung: create, then update if it was already there + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Regressions/LogMessageTests.cs b/tests/Xrm.Shuffle.Core.Tests/Regressions/LogMessageTests.cs new file mode 100644 index 0000000..90dc6ae --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Regressions/LogMessageTests.cs @@ -0,0 +1,207 @@ +namespace Cinteros.Crm.Utils.Shuffle.Tests.Regressions +{ + using System.ServiceModel; + using Cinteros.Crm.Utils.Shuffle.Tests.Helpers; + using Microsoft.Xrm.Sdk; + using NUnit.Framework; + + /// + /// Failures that leave the block by a route the per-record catch cannot see. + /// + /// + /// + /// ImportDataBlock wraps each record in a try, counts a failure, names the record and carries + /// on. That is the whole error story for anything a record does on its own. Batching adds two + /// routes around it. The delete-all pass runs before the record loop opens, and the three + /// flushes that empty the pending batches run after it closes, so a throw from either lands + /// outside the try and takes the whole block with it. + /// + /// + /// The other half is what the catch prints when it does fire. A flush that faults in the + /// middle of the loop is reported against whichever record happened to fill the batch, not + /// the one that faulted, unless the flush left a label behind. StopOnBatchError is what + /// leaves that label; where nothing sets one, the wrong record gets the blame and the tests + /// below say so plainly. + /// + /// + [TestFixture] + public class LogMessageTests : FakeOrgTestBase + { + private static Types.DataBlock DeleteEverything() + { + return DefinitionXml.DataBlock("Accounts", "account") + .BatchSize(10) + .ImportAttribute("Delete", "All") + .DeserializeBlock(); + } + + private static Types.DataBlock CreateInBatchesOf(int batchSize) + { + return DefinitionXml.DataBlock("Accounts", "account") + .BatchSize(batchSize) + .ImportAttribute("UpdateIdentical", "true") + .MatchOn("name") + .DeserializeBlock(); + } + + private static EntityCollection Sources(params string[] names) + { + var entities = new EntityCollection { EntityName = "account" }; + var seed = 1; + foreach (var name in names) + { + entities.Entities.Add(Record("account", Id(seed++), name)); + } + return entities; + } + + private static EntityCollection NoSources() + { + return new EntityCollection { EntityName = "account" }; + } + + #region Deletes that escape the block + + /// + /// The delete-all pass sits above the record loop, so the rethrow in the sequential + /// fallback has nothing to catch it. Nothing after it in the block runs. + /// + [Test] + public void A_delete_fault_that_is_not_a_missing_record_escapes_the_block_entirely() + { + Online().WithEntity( + Seeded("account", Id(101), "Alpha"), + Seeded("account", Id(102), "Beta")); + Service + .ThrowAlways("ExecuteMultiple", ExecuteMultipleResponseBuilder.Faulted("Batch delete refused")) + .ThrowAlways("Delete", ExecuteMultipleResponseBuilder.Faulted("Privilege denied")); + + var shuffler = NewShuffler(); + + Assert.Throws>( + () => shuffler.TestImportDataBlock(DeleteEverything(), NoSources()), + DumpAll()); + Recorder.AssertLogged("Falling back to sequential deletes"); + Recorder.AssertNeverSent("*** Error record"); + } + + /// + /// The one fault the fallback does swallow, because a cascade delete in the target may + /// legitimately have taken the record already. It is not counted as a delete either. + /// + [Test] + public void A_record_that_was_already_gone_is_tolerated_by_the_same_fallback() + { + Online().WithEntity( + Seeded("account", Id(101), "Alpha"), + Seeded("account", Id(102), "Beta")); + Service + .ThrowAlways("ExecuteMultiple", ExecuteMultipleResponseBuilder.Faulted("Batch delete refused")) + .ThrowAlways("Delete", ExecuteMultipleResponseBuilder.Faulted( + "account With Id = 00000000-0000-0000-0000-000000000065 Does Not Exist")); + + var outcome = NewShuffler().TestImportDataBlock(DeleteEverything(), NoSources()); + + Assert.That(outcome.Deleted, Is.EqualTo(0), DumpAll()); + Assert.That(outcome.Failed, Is.EqualTo(0), DumpAll()); + Recorder.AssertSent("...already deleted"); + } + + /// + /// StopOnError does not reach the delete-all pass at all - the rethrow is unconditional, + /// so the block ends the same way whether the definition asked to stop or not. + /// + [Test] + public void StopOnError_makes_no_difference_to_a_delete_that_escapes() + { + Online().WithEntity( + Seeded("account", Id(101), "Alpha"), + Seeded("account", Id(102), "Beta")); + Service + .ThrowAlways("ExecuteMultiple", ExecuteMultipleResponseBuilder.Faulted("Batch delete refused")) + .ThrowAlways("Delete", ExecuteMultipleResponseBuilder.Faulted("Privilege denied")); + + var shuffler = NewShuffler(stopOnError: false); + + Assert.Throws>( + () => shuffler.TestImportDataBlock(DeleteEverything(), NoSources()), + DumpAll()); + Assert.That(shuffler.TestBatchFailureLabel, Is.Null, DumpAll()); + } + + #endregion Deletes that escape the block + + #region Flushes that escape the block + + /// + /// The three flushes at the end of the block run after the record loop has closed, so a + /// StopOnError rethrow from one of them is never seen by the per-record catch and no + /// record is named at all. + /// + [Test] + public void An_end_of_block_flush_that_fails_under_StopOnError_escapes_without_naming_a_record() + { + // On-prem: no bulk messages, so the batch goes out as ExecuteMultiple and that is + // the rung whose catch rethrows under StopOnError. + OnPrem().WithMetadata("account"); + Service.ThrowAlways( + "ExecuteMultiple", ExecuteMultipleResponseBuilder.Faulted("Batch create refused")); + + var shuffler = NewShuffler(stopOnError: true); + + Assert.Throws>( + () => shuffler.TestImportDataBlock(CreateInBatchesOf(10), Sources("Alpha", "Beta")), + DumpAll()); + Recorder.AssertLogged("ExecuteMultiple batch create failed"); + Recorder.AssertNeverSent("*** Error record"); + Assert.That(shuffler.TestBatchFailureLabel, Is.Null, DumpAll()); + } + + /// + /// Without StopOnError the same two faults are contained: the flush drops to one create + /// per record and the block finishes normally. + /// + [Test] + public void Without_StopOnError_the_same_failure_drops_to_one_create_per_record() + { + // On-prem: no bulk messages, so the batch goes out as ExecuteMultiple and that is + // the rung whose catch rethrows under StopOnError. + OnPrem().WithMetadata("account"); + Service.ThrowAlways( + "ExecuteMultiple", ExecuteMultipleResponseBuilder.Faulted("Batch create refused")); + + var outcome = NewShuffler().TestImportDataBlock(CreateInBatchesOf(10), Sources("Alpha", "Beta")); + + Assert.That(outcome.Created, Is.EqualTo(2), DumpAll()); + Assert.That(outcome.Failed, Is.EqualTo(0), DumpAll()); + Recorder.AssertLogged("Falling back to sequential creates"); + Assert.That(Service.Created.Count, Is.EqualTo(2), DumpAll()); + } + + /// + /// A flush in the middle of the loop is inside the try, so the catch does fire - and with + /// no label set it blames the record that filled the batch. Both records were lost here, + /// and only the second one is named. + /// + [Test] + public void A_mid_loop_flush_failure_is_blamed_on_the_record_that_filled_the_batch() + { + // On-prem: no bulk messages, so the batch goes out as ExecuteMultiple and that is + // the rung whose catch rethrows under StopOnError. + OnPrem().WithMetadata("account"); + Service.ThrowAlways( + "ExecuteMultiple", ExecuteMultipleResponseBuilder.Faulted("Batch create refused")); + + var shuffler = NewShuffler(stopOnError: true); + + Assert.Throws>( + () => shuffler.TestImportDataBlock( + CreateInBatchesOf(2), Sources("Alpha", "Beta", "Gamma")), + DumpAll()); + Recorder.AssertSent("*** Error record: Beta ***"); + Recorder.AssertNeverSent("*** Error record: Alpha ***"); + } + + #endregion Flushes that escape the block + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Regressions/SendTextFormattingTests.cs b/tests/Xrm.Shuffle.Core.Tests/Regressions/SendTextFormattingTests.cs new file mode 100644 index 0000000..e6a66c0 --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Regressions/SendTextFormattingTests.cs @@ -0,0 +1,136 @@ +namespace Cinteros.Crm.Utils.Shuffle.Tests.Regressions +{ + using System; + using System.Linq; + using Cinteros.Crm.Utils.Shuffle.Tests.Helpers; + using NUnit.Framework; + + /// + /// What reaches the log when a record identifier happens to look like a format string. + /// + /// + /// + /// Every progress line in import and export goes through SendText, which formats the message + /// with its arguments and then hands the result to two places - the log, and the ShuffleEvent + /// stream the UI renders. Before this PR it handed the arguments along with it, so the logger + /// formatted the already-formatted string a second time. + /// + /// + /// That was invisible for as long as no record carried a brace. It is not invisible for a + /// record named after a template: an identifier holding a placeholder was rewritten with a + /// value from the same line, and one holding a lone brace threw FormatException out of a log + /// call, failing an import for a reason that had nothing to do with the data. + /// + /// + [TestFixture] + public class SendTextFormattingTests : ShuffleTestBase + { + [Test] + public void Arguments_are_applied_to_the_format_string() + { + NewShuffler().TestSendLine("{0:000} Created: {1}", 1, "Alpha"); + + Recorder.AssertLogged("001 Created: Alpha"); + } + + /// + /// The regression. The identifier carries a placeholder of its own, and the second format + /// pass filled it in from this same line - so the log claimed a record name that never + /// existed in the source file. + /// + [Test] + public void A_value_containing_a_placeholder_is_logged_verbatim() + { + NewShuffler().TestSendLine("{0:000} Created: {1}", 1, "Batch {0} rows"); + + Recorder.AssertLogged("001 Created: Batch {0} rows"); + Recorder.AssertNeverLogged("Batch 1 rows"); + } + + /// + /// The louder half of the same bug: a lone brace is not a placeholder, so the second pass + /// threw rather than misreporting. The throw escaped into the per-record catch and the + /// record was counted as failed. + /// + [Test] + public void A_value_containing_an_unmatched_brace_does_not_throw() + { + var shuffler = NewShuffler(); + + Assert.DoesNotThrow(() => shuffler.TestSendLine("{0:000} Created: {1}", 1, "Rate { high")); + + Recorder.AssertLogged("001 Created: Rate { high"); + } + + [Test] + public void A_value_containing_a_placeholder_reaches_the_event_stream_unchanged() + { + NewShuffler().TestSendLine("{0:000} Created: {1}", 1, "Batch {0} rows"); + + Recorder.AssertSent("001 Created: Batch {0} rows"); + Recorder.AssertNeverSent("Batch 1 rows"); + } + + [Test] + public void A_message_with_no_arguments_is_logged_as_written() + { + NewShuffler().TestSendLine("Pre-retrieved 12 records for matching"); + + Recorder.AssertLogged("Pre-retrieved 12 records for matching"); + } + + /// + /// Landmine marker, not a fix. The first format pass still runs when there are no + /// arguments at all, so an already-interpolated message carrying a brace throws. + /// + /// + /// This is reachable today: the per-record catch in ImportDataBlock reports failures as + /// an interpolated string built from the record identifier, with no arguments. A record + /// whose identifier holds a brace therefore throws out of the error report itself. If a + /// later change makes SendText skip the format pass for an empty argument list, this test + /// starts failing - and the right answer then is to delete it, not to restore the throw. + /// + [Test] + public void An_interpolated_message_carrying_a_brace_still_throws_with_no_arguments() + { + var shuffler = NewShuffler(); + + Assert.Throws(() => shuffler.TestSendLine("*** Error record: Rate { high ***")); + } + + /// + /// The length guard in SendText drops anything shorter than two characters, and a bare + /// newline is exactly one - so the blank lines that space the log out never reach it. + /// + [Test] + public void A_bare_newline_reaches_the_event_stream_but_not_the_log() + { + NewShuffler().TestSendBlankLine(); + + Assert.That(Recorder.Logger.Messages.Count, Is.EqualTo(0), DumpAll()); + Assert.That(Recorder.Events.Count, Is.EqualTo(1), DumpAll()); + } + + /// + /// One call, two events: the text and the newline that terminates it. Worth pinning + /// because the UI counts events, and a change that merged the two would halve them. + /// + [Test] + public void SendLine_raises_the_text_and_the_newline_as_two_events() + { + NewShuffler().TestSendLine("{0:000} Created: {1}", 1, "Alpha"); + + Assert.That(Recorder.Events.Count, Is.EqualTo(2), DumpAll()); + Assert.That(Recorder.Logger.Messages.Count, Is.EqualTo(1), DumpAll()); + } + + [Test] + public void The_log_and_the_event_stream_agree_on_the_formatted_text() + { + NewShuffler().TestSendLine("{0:000} Created: {1}", 1, "Batch {0} rows"); + + var logged = Recorder.Logger.Messages.First(); + Assert.That(Recorder.SentMessages, Has.Member(logged), DumpAll()); + } + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/SmokeTests.cs b/tests/Xrm.Shuffle.Core.Tests/SmokeTests.cs new file mode 100644 index 0000000..f9ef713 --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/SmokeTests.cs @@ -0,0 +1,95 @@ +using System.Linq; +using System.Reflection; +using System.Xml; +using System.Xml.Schema; +using Cinteros.Crm.Utils.Shuffle.Types; +using NUnit.Framework; + +namespace Cinteros.Crm.Utils.Shuffle.Tests +{ + /// + /// Proves the shuffle core compiles into a test assembly without WinForms or + /// XrmToolBox, that the embedded schemas travel with it, and that the batch size + /// default agrees across the three places that declare it. + /// + [TestFixture] + public class SmokeTests + { + private const string MinimalDefinition = + "" + + "" + + "" + + "" + + "" + + "" + + ""; + + private static XmlDocument Load(string xml) + { + var doc = new XmlDocument(); + doc.LoadXml(xml); + return doc; + } + + [Test] + public void The_definition_schemas_are_embedded_in_the_test_assembly() + { + // ShuffleHelper resolves them off Assembly.GetExecutingAssembly(), which is + // this assembly once the shared project is compiled in. If the .projitems + // ever stops carrying the EmbeddedResource items, validation silently + // becomes a no-op rather than failing - hence the explicit check. + var names = Assembly.GetExecutingAssembly().GetManifestResourceNames() + .Select(n => n.ToLowerInvariant()).ToList(); + + Assert.That(names.Any(n => n.EndsWith("shuffledefinition.xsd")), Is.True, + "ShuffleDefinition.xsd is not embedded: " + string.Join(", ", names)); + Assert.That(names.Any(n => n.EndsWith("queryexpression.xsd")), Is.True, + "QueryExpression.xsd is not embedded: " + string.Join(", ", names)); + } + + [Test] + public void A_minimal_definition_validates_against_the_schema() + { + Assert.That(() => ShuffleHelper.ValidateDefinitionXml(Load(MinimalDefinition)), + Throws.Nothing); + } + + [Test] + public void An_undeclared_attribute_fails_validation() + { + // Guards against the schemas failing to load: ValidateDefinitionXml returns + // quietly when fewer than two are registered, so a passing "valid" test on + // its own proves nothing. + var invalid = MinimalDefinition.Replace(" ShuffleHelper.ValidateDefinitionXml(Load(invalid)), + Throws.InstanceOf()); + } + + [Test] + public void Default_batch_size_is_one() + { + Assert.That(new DataBlockImport().BatchSize, Is.EqualTo(1)); + } + + [Test] + public void A_definition_that_does_not_mention_batch_size_gets_one() + { + var shuffler = new Shuffler(null) { Definition = Load(MinimalDefinition) }; + + Assert.That(shuffler.ShuffleDefinition.Blocks.Items[0], Is.InstanceOf()); + Assert.That(((DataBlock)shuffler.ShuffleDefinition.Blocks.Items[0]).Import.BatchSize, + Is.EqualTo(1)); + } + + [Test] + public void A_definition_that_opts_in_keeps_its_batch_size() + { + var opted = MinimalDefinition.Replace(" + /// Exercises the test doubles themselves. + /// + /// + /// The doubles encode assumptions about the product — the shape of the capability probe, that + /// ExecuteMultipleResponseItem is constructible, that a partial class reaches the private + /// members. If one of those stops holding, these tests are where it shows, rather than as a + /// confusing failure in a fixture that was testing something else entirely. + /// + [TestFixture] + public class TestDoubleTests : ShuffleTestBase + { + [Test] + public void Logger_records_messages_and_section_depth() + { + Container.Logger.StartSection("Outer"); + Container.Logger.Log("Block: {0}", "Accounts"); + Container.Logger.EndSection(); + + Assert.That(Recorder.Logger.Logged("Block: Accounts"), Is.True, Recorder.Logger.Dump()); + Assert.That(Recorder.Logger.SectionDepth, Is.EqualTo(0)); + } + + [Test] + public void Scripted_service_records_every_request_in_order() + { + Service.OnCreate(entity => Id(7)); + Service.Create(new Entity("account")); + Service.Update(new Entity("account", Id(7))); + Service.Delete("account", Id(7)); + + Assert.That(Service.RequestNames, Is.EqualTo(new[] { "Create", "Update", "Delete" })); + Assert.That(Service.Created.Single().LogicalName, Is.EqualTo("account")); + Assert.That(Service.Deleted.Single().Item2, Is.EqualTo(Id(7))); + } + + [Test] + public void Scripted_service_answers_the_capability_probe_only_for_declared_messages() + { + Service.SupportsBulkMessage("account", "CreateMultiple"); + var shuffler = NewShuffler(); + + Assert.That(shuffler.TestIsCreateMultipleSupported("account"), Is.True); + Assert.That(shuffler.TestIsUpdateMultipleSupported("account"), Is.False); + Assert.That(shuffler.TestIsCreateMultipleSupported("contact"), Is.False); + } + + [Test] + public void Unscripted_message_throws_naming_what_was_seen() + { + var thrown = Assert.Throws( + () => Service.Execute(new OrganizationRequest("CreateMultiple"))); + + Assert.That(thrown.Message, Does.Contain("CreateMultiple")); + } + + [Test] + public void Sequenced_responses_are_handed_out_once_each() + { + Service.OnMessageSequence( + "WhoAmI", + new OrganizationResponse { ResponseName = "first" }, + new OrganizationResponse { ResponseName = "second" }); + + Assert.That(Service.Execute(new OrganizationRequest("WhoAmI")).ResponseName, Is.EqualTo("first")); + Assert.That(Service.Execute(new OrganizationRequest("WhoAmI")).ResponseName, Is.EqualTo("second")); + Assert.Throws(() => Service.Execute(new OrganizationRequest("WhoAmI"))); + } + + [Test] + public void Omit_produces_a_response_collection_shorter_than_the_request_list() + { + var response = new ExecuteMultipleResponseBuilder() + .CreatedAt(Id(1)) + .Omit() + .CreatedAt(Id(3)) + .Build(); + + var items = (ExecuteMultipleResponseItemCollection)response.Results["Responses"]; + + Assert.That(items.Count, Is.EqualTo(2), "Omit must skip the item, not blank it out"); + Assert.That(items.Select(i => i.RequestIndex), Is.EqualTo(new[] { 0, 2 })); + Assert.That((bool)response.Results["IsFaulted"], Is.False); + } + + [Test] + public void A_fault_marks_the_whole_response_faulted() + { + var response = new ExecuteMultipleResponseBuilder() + .Succeeded() + .Fault("record is busy") + .Build(); + + Assert.That((bool)response.Results["IsFaulted"], Is.True); + } + + [Test] + public void Shim_reaches_the_private_state_the_import_sets_up_in_ImportToCRM() + { + var shuffler = NewShuffler(stopOnError: true); + + Assert.That(shuffler.TestGuidMap, Is.Not.Null, "guidmap is only assigned inside ImportToCRM"); + Assert.That(shuffler.TestStopOnError, Is.True); + Assert.That(shuffler.TestStopOnBatchError(3, "account Acme"), Is.True); + Assert.That(shuffler.TestBatchFailureLabel, Is.EqualTo("003 account Acme")); + } + + [Test] + public void Definition_literals_deserialize_into_the_objects_the_import_reads() + { + var block = DefinitionXml + .DataBlock("Accounts", "account") + .BatchSize(50) + .MatchOn("name") + .DeserializeBlock(); + + Assert.That(block.Name, Is.EqualTo("Accounts")); + Assert.That(block.Import.BatchSize, Is.EqualTo(50)); + Assert.That(block.Import.Match.PreRetrieveAll, Is.True); + Assert.That(block.Import.Match.Attribute.Single().Name, Is.EqualTo("name")); + } + + [Test] + public void An_omitted_BatchSize_deserializes_to_the_no_batching_default() + { + var block = DefinitionXml.DataBlock("Accounts", "account").Import().DeserializeBlock(); + + Assert.That(block.Import.BatchSize, Is.EqualTo(1)); + } + + [Test] + public void Simple_data_literals_deserialize_without_touching_the_service() + { + var data = DataXml + .Block("Accounts") + .Record("account", Id(1)).With("name", "Acme").WithInt("numberofemployees", 42) + .AndRecord("account", Id(2)).With("name", "Globex") + .Build(); + + var blocks = NewShuffler().Deserialize(Container, data); + var entities = blocks["Accounts"]; + + Assert.That(entities.Entities.Count, Is.EqualTo(2)); + Assert.That(entities.Entities[0].Id, Is.EqualTo(Id(1))); + Assert.That(entities.Entities[0]["name"], Is.EqualTo("Acme")); + Assert.That(entities.Entities[0]["numberofemployees"], Is.EqualTo(42)); + Assert.That(Service.Requests, Is.Empty, "Simple serialization must need no metadata"); + } + } +} diff --git a/tests/Xrm.Shuffle.Core.Tests/Xrm.Shuffle.Core.Tests.csproj b/tests/Xrm.Shuffle.Core.Tests/Xrm.Shuffle.Core.Tests.csproj new file mode 100644 index 0000000..fd32891 --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/Xrm.Shuffle.Core.Tests.csproj @@ -0,0 +1,109 @@ + + + + + Debug + AnyCPU + {7C3E9A21-4B8D-4E52-9F1C-6D0A5B2E4417} + Library + Properties + Xrm.Shuffle.Core.Tests + Xrm.Shuffle.Core.Tests + v4.8 + 512 + + true + true + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1.58.1 + + + 9.0.2.60 + + + 9.0.2.60 + + + 9.1.1.45 + + + 3.14.0 + + + 4.5.0 + + + + + + diff --git a/tests/Xrm.Shuffle.Core.Tests/app.config b/tests/Xrm.Shuffle.Core.Tests/app.config new file mode 100644 index 0000000..d4ceb94 --- /dev/null +++ b/tests/Xrm.Shuffle.Core.Tests/app.config @@ -0,0 +1,9 @@ + + + + + + + + +