Performance and usability enhancements, bug fixes, and CI/CD automation for Shuffle

.github/workflows/build.yml

@@ -0,0 +1,144 @@
+name: CI Build
+
+on:
+  push:
+    branches:
+      - '**'
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  build:
+    runs-on: windows-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Calculate version
+        id: version
+        shell: pwsh
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          $year  = (Get-Date).Year
+          $month = (Get-Date).Month
+          $startOfMonth = (Get-Date -Day 1 -Hour 0 -Minute 0 -Second 0).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")
+
+          $headers = @{
+            Authorization = "Bearer $env:GITHUB_TOKEN"
+            Accept        = "application/vnd.github.v3+json"
+          }
+          $url = "https://api.github.com/repos/$env:GITHUB_REPOSITORY/actions/workflows/build.yml/runs?created=>=$startOfMonth&per_page=1"
+          $response = Invoke-RestMethod -Uri $url -Headers $headers -ErrorAction Stop
+          $rev = $response.total_count
+
+          $version = "1.$year.$month.$rev"
+          "VERSION=$version" | Out-File -FilePath $env:GITHUB_OUTPUT -Append
+          Write-Host "Computed version: $version"
+
+      - name: Patch AssemblyInfo.cs
+        shell: pwsh
+        run: |
+          $version = "${{ steps.version.outputs.VERSION }}"
+          $file = "XTB\Properties\AssemblyInfo.cs"
+          (Get-Content $file) `
+            -replace 'AssemblyVersion\("[^"]*"\)',     "AssemblyVersion(`"$version`")" `
+            -replace 'AssemblyFileVersion\("[^"]*"\)', "AssemblyFileVersion(`"$version`")" |
+            Set-Content $file
+          Write-Host "Patched $file to version $version"
+
+      - name: Patch nuspec versions
+        shell: pwsh
+        run: |
+          $version = "${{ steps.version.outputs.VERSION }}"
+          $nuspecs = @(
+            "XTB\ShuffleRunner.nuspec",
+            "XTB\ShuffleBuilder.nuspec",
+            "XTB\ShuffleDeployer.nuspec"
+          )
+          foreach ($nuspec in $nuspecs) {
+            $path = Resolve-Path $nuspec
+            [xml]$xml = Get-Content $path
+            $xml.package.metadata.version = $version
+            $xml.Save($path)
+            Write-Host "Patched $nuspec to version $version"
+          }
+
+      - name: Setup NuGet
+        uses: NuGet/setup-nuget@v2
+
+      - name: NuGet restore
+        run: nuget restore Rappen.XTB.Shuffle.sln
+
+      - name: Setup MSBuild
+        uses: microsoft/setup-msbuild@v2
+
+      - name: Build solution (Release)
+        run: msbuild Rappen.XTB.Shuffle.sln /p:Configuration=Release /p:Platform="Any CPU" /m
+
+      - name: Locate VSTest
+        id: vstest
+        shell: pwsh
+        run: |
+          $vswhere = Join-Path ${env:ProgramFiles(x86)} "Microsoft Visual Studio\Installer\vswhere.exe"
+          if (-not (Test-Path $vswhere)) { throw "vswhere.exe not found at $vswhere" }
+          # No -requires / -latest: a bare -find lists only the instances that actually
+          # contain the file, so Build Tools counts and the newest instance without the
+          # test platform cannot win and yield nothing.
+          $console = & $vswhere -products * `
+                       -find "Common7\IDE\Extensions\TestPlatform\vstest.console.exe" | Select-Object -First 1
+          if (-not $console) { throw "vstest.console.exe not found by vswhere" }
+          "PATH=$console" | Out-File -FilePath $env:GITHUB_OUTPUT -Append
+          Write-Host "Found $console"
+
+      - name: Run tests (Release)
+        shell: pwsh
+        run: |
+          & "${{ steps.vstest.outputs.PATH }}" `
+            "tests\Xrm.Shuffle.Core.Tests\bin\Release\Xrm.Shuffle.Core.Tests.dll" `
+            /Framework:.NETFramework,Version=v4.8 `
+            /Logger:"trx;LogFileName=Xrm.Shuffle.Core.Tests.Release.trx" `
+            /ResultsDirectory:TestResults
+          if ($LASTEXITCODE -ne 0) { throw "Tests failed with exit code $LASTEXITCODE" }
+
+      # The product compiles differently in the two configurations: ShuffleDataImport
+      # logs the match query as FetchXml under #if DEBUG, which sends a message the
+      # test doubles have to answer. Release alone would let that divergence through,
+      # green here and red in Visual Studio, which is what happened once already.
+      - name: Build solution (Debug)
+        run: msbuild Rappen.XTB.Shuffle.sln /p:Configuration=Debug /p:Platform="Any CPU" /m
+
+      - name: Run tests (Debug)
+        shell: pwsh
+        run: |
+          & "${{ steps.vstest.outputs.PATH }}" `
+            "tests\Xrm.Shuffle.Core.Tests\bin\Debug\Xrm.Shuffle.Core.Tests.dll" `
+            /Framework:.NETFramework,Version=v4.8 `
+            /Logger:"trx;LogFileName=Xrm.Shuffle.Core.Tests.Debug.trx" `
+            /ResultsDirectory:TestResults
+          if ($LASTEXITCODE -ne 0) { throw "Tests failed with exit code $LASTEXITCODE" }
+
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-results-${{ steps.version.outputs.VERSION }}
+          path: TestResults/*.trx
+
+      - name: NuGet pack
+        shell: pwsh
+        run: |
+          New-Item -ItemType Directory -Force -Path nupkg | Out-Null
+          nuget pack "XTB\ShuffleRunner.nuspec" -OutputDirectory nupkg
+          nuget pack "XTB\ShuffleBuilder.nuspec" -OutputDirectory nupkg
+          nuget pack "XTB\ShuffleDeployer.nuspec" -OutputDirectory nupkg
+
+      - name: Upload nupkg artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: nupkg-${{ steps.version.outputs.VERSION }}
+          path: nupkg/*.nupkg

.github/workflows/release.yml

@@ -0,0 +1,44 @@
+name: Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      run_id:
+        description: 'CI run ID whose nupkg artifact to publish (find it in the Actions tab URL)'
+        required: true
+
+jobs:
+  release:
+    runs-on: windows-latest
+
+    steps:
+      - name: Download nupkg artifact from CI run
+        uses: actions/download-artifact@v4
+        with:
+          run-id: ${{ inputs.run_id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          pattern: 'nupkg-*'
+          path: nupkg
+          merge-multiple: true
+
+      - name: Resolve version from nupkg filename
+        id: version
+        shell: pwsh
+        run: |
+          $pkg = Get-ChildItem nupkg\*.nupkg | Select-Object -First 1
+          if (-not $pkg) { throw "No .nupkg found in artifact" }
+          # filename example: Rappen.XrmToolBox.Shuffle.Runner.1.2026.4.5.nupkg
+          $version = $pkg.BaseName -replace '^Rappen\.XrmToolBox\.Shuffle\.\w+\.', ''
+          "VERSION=$version" | Out-File -FilePath $env:GITHUB_OUTPUT -Append
+          Write-Host "Releasing version: $version  ($($pkg.Name))"
+
+      - name: Publish to NuGet.org
+        run: nuget push nupkg\*.nupkg -ApiKey ${{ secrets.NUGET_API_KEY }} -Source https://api.nuget.org/v3/index.json -NonInteractive
+
+      - name: Create GitHub release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: v${{ steps.version.outputs.VERSION }}
+          name: Release ${{ steps.version.outputs.VERSION }}
+          generate_release_notes: true
+          files: nupkg/*.nupkg

.gitignore

@@ -257,3 +257,8 @@ VSIX/
 pat.txt
 test.txt
 /codealike.json
+/.claude/settings.local.json
+
+# Test run output
+TestResults/
+*.trx

CLAUDE.md

@@ -0,0 +1,151 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## What This Project Is
+
+Xrm.Shuffle is an **XrmToolBox plugin suite** for Dataverse/Dynamics CRM. It ships as three separate NuGet packages — Shuffle Builder, Shuffle Runner, Shuffle Deployer — all compiled from a **single assembly** (`Rappen.XTB.Shuffle.dll`). The tools read/write XML-based "ShuffleDefinition" files that describe what Dataverse data or solutions to export or import.
+
+## Build
+
+```bash
+nuget restore Rappen.XTB.Shuffle.sln
+msbuild Rappen.XTB.Shuffle.sln /p:Configuration=Release /p:Platform="Any CPU" /m
+```
+
+Output goes to `XTB\bin\Release\`.
+
+Unit tests live in `tests\Xrm.Shuffle.Core.Tests\`. That project imports the same two
+`.projitems` as the XTB project, so it compiles the shuffle core without WinForms or
+XrmToolBox. Build the **solution** rather than the csproj (the project platform is
+`AnyCPU` and the sln does the `Any CPU` mapping), then run the tests with the VSTest
+console:
+
+```bash
+vstest.console.exe "tests\Xrm.Shuffle.Core.Tests\bin\Release\Xrm.Shuffle.Core.Tests.dll" /Framework:.NETFramework,Version=v4.8
+```
+
+NUnit3TestAdapter arrives through `PackageReference` and is auto-imported, so no
+`/TestAdapterPath` is needed.
+
+Run **both** configurations. `ShuffleDataImport` logs the match query as FetchXml
+inside `#if DEBUG`, which sends a message the test doubles have to answer, so Debug
+exercises calls Release never makes - and Visual Studio Test Explorer defaults to
+Debug. `.github/workflows/build.yml` builds and tests both, and fails the build on a
+red test in either.
+
+Everything outside that project - solution import and export, data export, and anything
+that needs a live org - is still validated manually.
+
+To produce NuGet packages:
+```bash
+nuget pack "XTB\ShuffleRunner.nuspec" -OutputDirectory nupkg
+nuget pack "XTB\ShuffleBuilder.nuspec" -OutputDirectory nupkg
+nuget pack "XTB\ShuffleDeployer.nuspec" -OutputDirectory nupkg
+```
+
+CI runs `.github/workflows/build.yml` / `release.yml` with versioning scheme `1.{year}.{month}.{run_number}`.
+
+## Architecture
+
+### Project layout
+
+```
+XTB/                        ← Single .csproj for all three tools
+  Builder/                  ← Visual XML editor for creating definitions
+  Runner/                   ← Executes definitions (export or import)
+  Deployer/                 ← Orchestrates packaged deployments (.cdpkg/.cdzip)
+shared/Xrm.Shuffle.Core/   ← Shared Project (.shproj) — compiled directly into XTB, no separate DLL
+Xrm.Utils.Core/            ← Git submodule: extensions, fluent API, logging, CSV helpers
+```
+
+### The Shared Project pattern
+
+`Xrm.Shuffle.Core` is a **Shared Project** (`.shproj`), not a library. Its files are compiled directly into the main project. All core business logic lives here:
+
+- **`Shuffler.cs`** — top-level orchestrator; parses the ShuffleDefinition XML, drives block execution, dispatches events
+- **`ShuffleDataImport.cs`** — high-performance importer with a capability-detection fallback chain (see below)
+- **`ShuffleDataExport.cs`** — query-based exporter; supports filter and FetchXML modes
+- **`ShuffleSolutionImport/Export.cs`** — solution package handling
+- **`Types.cs`** — core enums (`SerializationType`, `ItemImportResult`, `SolutionImportConditions`)
+- **`ShuffleHelper.cs`** — schema validation, DataFileRequired checks, node documentation lookup
+- **`Const.cs`** — auto-generated latebound constants for CRM API entities (ImportJob, AsyncOperation)
+
+The XML schema is `Resources/ShuffleDefinition.xsd`; the corresponding C# class `Resources/ShuffleDefinition.cs` is auto-generated from it.
+
+### Bulk import strategy — runtime capability detection
+
+`ShuffleDataImport.cs` detects what the connected environment supports **at runtime per entity** (cached) and walks down this chain:
+
+1. **UpsertMultiple** (Dataverse online, fastest — bypasses PreRetrieveAll entirely when `UpdateIdentical=true`)
+2. **CreateMultiple / UpdateMultiple** (Dataverse online)
+3. **ExecuteMultipleRequest** (CRM 9.1 on-premises fallback)
+4. **Individual operations** (CRM 8.x fallback)
+
+Detection uses `sdkmessagefilter` queries. Results are cached per entity name to avoid repeated round-trips.
+
+### Batching
+
+Creates, updates and upserts are accumulated into a pending list and flushed in
+batches of `BatchSize` (default 1, i.e. no batching). The batch must be flushed early whenever the
+next step needs the server to already know about the pending records — before a live
+`Match` query, and before `ReplaceGuids` rewrites a record whose lookups point at a
+record still in the batch. `IsBatchable` decides what may be batched at all; among
+other things it excludes any record carrying `statecode`, `statuscode` or `ownerid`.
+
+### DeferStateAndOwner
+
+When `DeferStateAndOwner` is enabled, `statecode`/`statuscode`/`ownerid` are stripped
+from the record before it is saved and applied in a second pass once the records
+exist. The point is the `IsBatchable` exclusion above: without deferring, every record
+that carries one of those attributes falls off the batched path and is imported one at
+a time, so a block of inactive or non-default-owner records gets no batching at all.
+Deferring keeps those records batchable and moves the state/owner work into a separate
+pass that can itself be batched. The end state of each record is unchanged.
+
+The option is ignored for a block whose records carry *nothing but* state and owner —
+a common pattern where the state changes live in their own `Save="UpdateOnly"` block.
+Stripping there would leave empty records to save, so `IsStateOwnerOnlyBlock` turns the
+option off for the block and logs that it did; `HasAttributesBesidesStateOwner` catches
+the same case per record in a mixed block.
+
+No benchmark numbers are published for this — the gain depends entirely on how many
+records in the block carry those attributes.
+
+### XrmToolBox plugin model
+
+All three UI classes inherit `PluginControlBase` and implement standard XrmToolBox interfaces (`IMessageBusHost`, `IGitHubPlugin`, `IHelpPlugin`, `IAboutPlugin`). Long-running operations raise events via `ShuffleEventHandler` / `ShuffleEventArgs` rather than blocking the UI thread.
+
+### Builder controls bind through `Tag`
+
+Each editor under `XTB/Builder/Controls/` derives from `ControlBase`, which reads and
+writes the definition XML purely by convention: every control whose `Tag` is set to
+`"AttributeName|required|defaultvalue"` becomes that XML attribute, ordered by
+`TabIndex`, and a value equal to the default is omitted from the output. Adding a new
+attribute to the schema therefore means adding a control with the matching `Tag` —
+there is no separate mapping table.
+
+`ControlBase` also fills one "Information" box per node from the `<xs:documentation>`
+on the *element* in `ShuffleDefinition.xsd`, via `ShuffleHelper.GetNodeDocumentation`.
+It does not resolve documentation on *attributes*, so per-attribute help has to be a
+tooltip on the control.
+
+### Serialization types
+
+Six types control export format: `Full`, `Simple`, `SimpleWithValue`, `SimpleNoId`, `Explicit`, `Text`. `SimpleWithValue` is the default for Runner. Attributes are sorted alphabetically on export for deterministic diffs.
+
+### Namespaces
+
+| Area | Namespace |
+|---|---|
+| Core logic | `Cinteros.Crm.Utils.Shuffle` |
+| Builder UI | `Rappen.XTB.Shuffle.Builder` |
+| Runner UI | `Rappen.XTB.Shuffle.Runner` |
+| Deployer UI | `Rappen.XTB.ShuffleDeployer` |
+
+## Key dependencies
+
+- **XrmToolBoxPackage** — plugin framework (PluginControlBase, connection management)
+- **Xrm.Utils.Core** (submodule) — IExecutionContainer, ILogger, entity/service extensions
+- **Microsoft.CrmSdk.Workflow** — CRM SDK types
+- **System.IO.Compression** — used by Deployer for `.cdpkg` (ZIP) handling