diff --git a/src/TALXIS.CLI.Features.Data/DataModelConvertCliCommand.cs b/src/TALXIS.CLI.Features.Data/DataModelConvertCliCommand.cs index a42eee0a..1f3b5e0f 100644 --- a/src/TALXIS.CLI.Features.Data/DataModelConvertCliCommand.cs +++ b/src/TALXIS.CLI.Features.Data/DataModelConvertCliCommand.cs @@ -1,6 +1,7 @@ -using DotMake.CommandLine; +using DotMake.CommandLine; using Microsoft.Extensions.Logging; using TALXIS.CLI.Features.Data.DataModelConverter; +using TALXIS.CLI.Features.Data.DataModelConverter.AppScope; using TALXIS.CLI.Core; using TALXIS.CLI.Logging; @@ -20,10 +21,32 @@ public class DataModelConvertCliCommand : TxcLeafCommand [CliOption( Name = "--input", Aliases = ["-i"], - Description = "Path to the input: a solution project folder (.cdsproj/.csproj with SolutionRootPath), a declarations folder, or a .zip solution file. Defaults to the current directory.", + Description = "Path to an input: a solution project folder (.cdsproj/.csproj with SolutionRootPath), a declarations folder, or a .zip solution file. Can be specified multiple times to merge several sources into one model; earlier inputs win where two declare the same attribute differently. Defaults to the current directory.", Required = false )] - public string? InputPath { get; set; } + public List InputPaths { get; set; } = []; + + [CliOption( + Name = "--root", + Description = "Path to a repository root; every declarations folder beneath it becomes an input. Can be specified multiple times. Use this rather than listing folders when a project's model spans many modules, and pass the product repository as a second root when the base model lives there.", + Required = false + )] + public List Roots { get; set; } = []; + + [CliOption( + Name = "--app", + Description = "Unique name of a model-driven app. Narrows the output to the tables that app is built on, instead of everything the inputs declare. App modules are searched for under --root, or under the inputs when no root is given.", + Required = false + )] + public string? AppUniqueName { get; set; } + + [CliOption( + Name = "--detail", + Description = "How much to emit. 'full' is everything the inputs declare. 'minimal' shows how the app was built: each table keeps only the columns its own forms, views, workflows, sitemap and .cs/.ts sources refer to, platform plumbing is dropped, and an N:N appears only when both its tables belong to the app. A dropped column is one no reference was found for, which is not the same as one that is unused: a name built at runtime cannot be found at all. 'minimal' requires --app, and is not a schema export -- use 'full' to generate SQL or EDMX for tooling.", + AllowedValues = new[] { "full", "minimal" }, + Required = false + )] + public string Detail { get; set; } = "full"; [CliOption( Name = "--target", @@ -43,7 +66,42 @@ public class DataModelConvertCliCommand : TxcLeafCommand protected override Task ExecuteAsync() { - var inputPath = InputPath ?? Directory.GetCurrentDirectory(); + var detail = string.Equals(Detail, "minimal", StringComparison.OrdinalIgnoreCase) + ? DetailLevel.Minimal + : DetailLevel.Full; + + if (detail == DetailLevel.Minimal && string.IsNullOrWhiteSpace(AppUniqueName)) + { + throw new ArgumentException("--detail minimal narrows an app's tables to how that app uses them, so it requires --app."); + } + + var inputPaths = new List(InputPaths); + + foreach (var root in Roots) + { + var discovered = DataModelConverterService.DiscoverDeclarationFolders(root); + if (discovered.Count == 0) + { + Logger.LogWarning("No declarations were found under root {Root}.", root); + } + inputPaths.AddRange(discovered); + } + + // Scoping to an app needs to reach the module that declares it, which is not the + // module that declares the entities -- so when only an app is named, search from + // the enclosing repository rather than the working directory alone. + var appSearchRoots = new List(Roots); + if (!string.IsNullOrWhiteSpace(AppUniqueName) && appSearchRoots.Count == 0) + { + var enclosing = FindEnclosingRepositoryRoot(Directory.GetCurrentDirectory()); + appSearchRoots.Add(enclosing); + Logger.LogInformation("Searching for app modules under {Root}.", enclosing); + } + + if (inputPaths.Count == 0) + { + inputPaths.Add(Directory.GetCurrentDirectory()); + } var outputDir = OutputDirectory ?? Path.Combine(Directory.GetCurrentDirectory(), ExportsFolderName); Directory.CreateDirectory(outputDir); @@ -52,12 +110,66 @@ protected override Task ExecuteAsync() var extension = TargetFormat!.ToLower() == "plainsql" ? "sql" : TargetFormat.ToLower(); var outputFilePath = Path.Combine(outputDir, $"solution.{extension}"); - DataModelConverterService.ConvertModel(inputPath, TargetFormat!, outputFilePath); + var droppedColumns = DataModelConverterService.ConvertModel( + inputPaths, TargetFormat!, outputFilePath, AppUniqueName, appSearchRoots, detail); + + var summary = new ConvertSummary( + outputFilePath, + Detail.ToLowerInvariant(), + droppedColumns.Count, + [.. droppedColumns.GroupBy(c => c.Reason) + .OrderBy(g => g.Key) + .Select(g => new DroppedByReason(g.Key.ToString(), g.Count()))], + droppedColumns); + + // Every dropped column in one warning is unreadable once there are thousands of + // them, so the full list goes to the data channel and text mode gets the counts. + OutputFormatter.WriteData(summary, s => + { + OutputWriter.WriteLine($"Output written to: {s.OutputFile}"); + foreach (var reason in s.DroppedByReason) + { + OutputWriter.WriteLine($" dropped {reason.Count} column(s): {reason.Reason}"); + } + }); - OutputFormatter.WriteResult("succeeded", $"Output written to: {outputFilePath}"); return Task.FromResult(ExitSuccess); } + /// Where the converted model was written. + /// The detail level the conversion ran at. + /// How many columns were left out in total. + /// Counts per reason, which is what a reader needs first. + /// Every dropped column, for a caller that wants to check one. + public sealed record ConvertSummary( + string OutputFile, + string Detail, + int ColumnsDropped, + IReadOnlyList DroppedByReason, + IReadOnlyList DroppedColumns); + + /// Why these columns were left out. + /// How many were left out for that reason. + public sealed record DroppedByReason(string Reason, int Count); + + /// + /// Walks up for the repository that encloses a directory, so an app can be found + /// without the caller naming a root. Falls back to the directory itself. + /// + private static string FindEnclosingRepositoryRoot(string startPath) + { + var dir = new DirectoryInfo(startPath); + while (dir != null) + { + if (Directory.Exists(Path.Combine(dir.FullName, ".git")) || dir.GetFiles("*.sln").Length > 0) + { + return dir.FullName; + } + dir = dir.Parent; + } + return startPath; + } + /// /// Ensures the exports folder is listed in the nearest .gitignore, /// adding an entry if it is not already present. diff --git a/src/TALXIS.CLI.Features.Data/DataModelConverter/AppScope/AppScopeFilter.cs b/src/TALXIS.CLI.Features.Data/DataModelConverter/AppScope/AppScopeFilter.cs new file mode 100644 index 00000000..9734cb52 --- /dev/null +++ b/src/TALXIS.CLI.Features.Data/DataModelConverter/AppScope/AppScopeFilter.cs @@ -0,0 +1,50 @@ +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Logging; +using TALXIS.CLI.Features.Data.DataModelConverter.Model; +using TALXIS.CLI.Logging; + +namespace TALXIS.CLI.Features.Data.DataModelConverter.AppScope; + +/// Narrows a parsed model to the tables an app is built on. +public static class AppScopeFilter +{ + private static readonly ILogger _logger = TxcLoggerFactory.CreateLogger(nameof(AppScopeFilter)); + + /// + /// Drops tables the app does not declare. Runs before relationships are built, so a + /// relationship out of the app cannot bring its referencing table back as a stub. A + /// relationship into a dropped table still stubs the far side deliberately, so + /// a lookup terminates somewhere visible — see + /// for how the diagram tells that apart from a table no input declares. + /// + public static void ApplyTableScope(List tables, ResolvedAppScope scope) + { + // The one point where every input's declarations are still present. + foreach (var table in tables.Where(t => t.Type == TableType.InSolution)) + { + scope.AllDeclaredTableLogicalNames.Add(table.LogicalName); + } + + var removed = tables.RemoveAll(t => + t.Type == TableType.InSolution && !scope.TableLogicalNames.Contains(t.LogicalName)); + + var missing = scope.TableLogicalNames + .Where(name => !tables.Any(t => string.Equals(t.LogicalName, name, System.StringComparison.OrdinalIgnoreCase))) + .OrderBy(x => x) + .ToList(); + + _logger.LogInformation( + "Scoped to app {App}: kept {Kept} table(s), dropped {Dropped} not declared by it.", + scope.UniqueName, tables.Count, removed); + + if (missing.Count > 0) + { + // The app names them but no input declares them — usually a module that was + // not passed in, which would otherwise show up only as an oddly small diagram. + _logger.LogWarning( + "App {App} references {Count} table(s) that none of the given inputs declare: {Tables}.", + scope.UniqueName, missing.Count, string.Join(", ", missing)); + } + } +} diff --git a/src/TALXIS.CLI.Features.Data/DataModelConverter/AppScope/AppScopeResolver.cs b/src/TALXIS.CLI.Features.Data/DataModelConverter/AppScope/AppScopeResolver.cs new file mode 100644 index 00000000..274385cc --- /dev/null +++ b/src/TALXIS.CLI.Features.Data/DataModelConverter/AppScope/AppScopeResolver.cs @@ -0,0 +1,196 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml.Linq; +using Microsoft.Extensions.Logging; +using TALXIS.CLI.Logging; + +namespace TALXIS.CLI.Features.Data.DataModelConverter.AppScope; + +/// The tables a model-driven app is built on, resolved from source. +public class ResolvedAppScope +{ + public string UniqueName { get; init; } = string.Empty; + + /// Compared case-insensitively: an app component's schemaName casing is not + /// guaranteed to match the casing of the entity's own declaration. + public HashSet TableLogicalNames { get; } = new(StringComparer.OrdinalIgnoreCase); + + /// Every file that contributed, for reporting which sources were read. + public List SourceFiles { get; } = []; + + /// How much metadata to emit. narrows the + /// columns of each table to what its own artefacts refer to. + public DetailLevel Detail { get; set; } = DetailLevel.Full; + + /// Where to look for references. Usually repository roots. + public List SearchRoots { get; set; } = []; + + /// Publisher customization prefixes of the inputs. A column carrying one was + /// created by an author, whatever its metadata says. + public HashSet AuthorPrefixes { get; } = new(StringComparer.OrdinalIgnoreCase); + + /// + /// Every table any input declares, captured before app scoping removes the ones this + /// app does not use. A stub for a table in here is outside the app, not outside the + /// solution — a distinction the diagram would otherwise get wrong. + /// + public HashSet AllDeclaredTableLogicalNames { get; } = new(StringComparer.OrdinalIgnoreCase); + + /// Columns removed, so the run can report them rather than drop them quietly. + public List DroppedColumns { get; } = []; +} + +/// +/// Resolves which tables an app declares, from the app module files on disk. No Dataverse +/// connection: the app's component list is in source, and so is everything it names. +/// +public static class AppScopeResolver +{ + private static readonly ILogger _logger = TxcLoggerFactory.CreateLogger(nameof(AppScopeResolver)); + + private const string AppModulesFolder = "AppModules"; + private const string SiteMapsFolder = "AppModuleSiteMaps"; + + /// The component type that carries a table's name. Views, forms, charts and + /// workflows reference their owning table only by id, so they cannot contribute one. + private const string EntityComponentType = "1"; + + public static ResolvedAppScope Resolve(IEnumerable searchRoots, string appUniqueName) + { + var byName = DiscoverAppModules(searchRoots); + + if (!byName.TryGetValue(appUniqueName, out var files) || files.Count == 0) + { + var known = byName.Keys.OrderBy(x => x, StringComparer.OrdinalIgnoreCase).ToList(); + throw new InvalidOperationException( + $"No app module named '{appUniqueName}' was found under the given inputs. " + + (known.Count == 0 + ? "No app modules were found at all — check that a repository root containing them was passed." + : $"Apps found: {string.Join(", ", known)}.")); + } + + var scope = new ResolvedAppScope { UniqueName = appUniqueName }; + + // One logical app can be declared across several files: a base declaration plus + // fragments contributed by other areas, whose components carry solutionaction="Added". + // The app's real component set is the union of all of them. + foreach (var file in files) + { + scope.SourceFiles.Add(file); + var doc = Load(file); + if (doc?.Root == null) continue; + + foreach (var component in doc.Root.Descendants("AppModuleComponent")) + { + if (component.Attribute("type")?.Value != EntityComponentType) continue; + var schemaName = component.Attribute("schemaName")?.Value; + if (!string.IsNullOrWhiteSpace(schemaName)) scope.TableLogicalNames.Add(schemaName); + } + } + + foreach (var table in ResolveSiteMapTables(searchRoots, appUniqueName)) + { + scope.TableLogicalNames.Add(table); + } + + _logger.LogInformation( + "App {App} resolves to {Count} tables, from {Files} declaration file(s).", + appUniqueName, scope.TableLogicalNames.Count, scope.SourceFiles.Count); + + return scope; + } + + /// + /// Every app module found, keyed by the UniqueName inside the file. The folder name is + /// only a locator — its casing can differ from the declared name, which matters on a + /// case-sensitive filesystem. + /// + public static Dictionary> DiscoverAppModules(IEnumerable searchRoots) + { + var byName = new Dictionary>(StringComparer.OrdinalIgnoreCase); + + foreach (var file in FilesUnder(searchRoots, AppModulesFolder, "AppModule*.xml")) + { + var uniqueName = Load(file)?.Root?.Element("UniqueName")?.Value; + if (string.IsNullOrWhiteSpace(uniqueName)) continue; + + if (!byName.TryGetValue(uniqueName, out var list)) + { + byName[uniqueName] = list = []; + } + list.Add(file); + } + + return byName; + } + + /// + /// Tables a sitemap surfaces. They appear either as an Entity attribute or as an etn + /// parameter inside a Url, and both forms occur in the same file. + /// + private static IEnumerable ResolveSiteMapTables(IEnumerable searchRoots, string appUniqueName) + { + foreach (var file in FilesUnder(searchRoots, SiteMapsFolder, "AppModuleSiteMap*.xml")) + { + var doc = Load(file); + if (doc?.Root == null) continue; + + var owner = doc.Root.Element("SiteMapUniqueName")?.Value; + if (!string.Equals(owner, appUniqueName, StringComparison.OrdinalIgnoreCase)) continue; + + foreach (var element in doc.Root.Descendants()) + { + var entity = element.Attribute("Entity")?.Value; + if (!string.IsNullOrWhiteSpace(entity)) yield return entity; + + var url = element.Attribute("Url")?.Value; + if (string.IsNullOrWhiteSpace(url)) continue; + + foreach (var part in url.Split('&', '?')) + { + if (part.StartsWith("etn=", StringComparison.OrdinalIgnoreCase) && part.Length > 4) + { + yield return part[4..]; + } + } + } + } + } + + /// + /// Anchors on the component folder rather than on "Declarations": some modules keep + /// their declarations under "CDS" instead, and a glob anchored on either name misses + /// the other. + /// + private static IEnumerable FilesUnder(IEnumerable searchRoots, string folderName, string pattern) + { + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var root in searchRoots.Where(Directory.Exists)) + { + foreach (var folder in Directory.EnumerateDirectories(root, folderName, SearchOption.AllDirectories)) + { + foreach (var file in Directory.EnumerateFiles(folder, pattern, SearchOption.AllDirectories)) + { + // Managed and unmanaged copies sit side by side with identical content. + if (seen.Add(Path.GetFullPath(file))) yield return file; + } + } + } + } + + private static XDocument? Load(string file) + { + try + { + return XDocument.Load(file); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Could not read app module {File}; skipping it.", file); + return null; + } + } +} diff --git a/src/TALXIS.CLI.Features.Data/DataModelConverter/AppScope/AttributeReferenceFilter.cs b/src/TALXIS.CLI.Features.Data/DataModelConverter/AppScope/AttributeReferenceFilter.cs new file mode 100644 index 00000000..00cad5d7 --- /dev/null +++ b/src/TALXIS.CLI.Features.Data/DataModelConverter/AppScope/AttributeReferenceFilter.cs @@ -0,0 +1,249 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using Microsoft.Extensions.Logging; +using TALXIS.CLI.Features.Data.DataModelConverter.Model; +using TALXIS.CLI.Logging; + +namespace TALXIS.CLI.Features.Data.DataModelConverter.AppScope; + +/// +/// Narrows each table to the columns its own artefacts mention. +/// +/// A reference belongs to the table whose artefact made it. Resolving one by name alone +/// keeps a column on every table that declares that name, which is a fair approximation for +/// an author's column — one table usually declares it — and useless for a platform column, +/// whose name is identical on every table in the org. One view showing createdon kept it on +/// nineteen tables in a real app. +/// +public static class AttributeReferenceFilter +{ + private static readonly ILogger _logger = TxcLoggerFactory.CreateLogger(nameof(AttributeReferenceFilter)); + + /// Identifier-shaped tokens; column logical names are always of this shape. + private static readonly Regex TokenPattern = new(@"[A-Za-z_][A-Za-z0-9_]{2,}", RegexOptions.Compiled); + + /// + /// Folders whose contents reference columns. Entity declarations are deliberately not + /// among them: an entity declares its own columns, so scanning one would report every + /// column as referenced by itself. + /// + private static readonly string[] ReferencingFolders = + ["FormXml", "SavedQueries", "Workflows", "Visualizations", "AppModuleSiteMaps", "AppModules"]; + + private static readonly string[] CodeExtensions = [".cs", ".ts", ".js"]; + + /// Business process flow bookkeeping. No metadata flag separates these from an + /// author's columns, so they are named. + private static readonly string[] ProcessFlowColumns = ["processid", "stageid", "traversedpath"]; + + private const string BaseCurrencySuffix = "_base"; + + public static void Apply(List
tables, List relationships, ResolvedAppScope scope) + { + var references = CollectReferences(scope.SearchRoots); + + // Computed before anything is dropped. The translators read + // Relationship.LeftSideRow/RighSideRow without a null check, so removing a row an + // edge points at turns a narrower diagram into a crash on the sql and edmx targets. + var loadBearing = new HashSet(); + foreach (var relationship in relationships) + { + if (relationship.LeftSideRow != null) loadBearing.Add(relationship.LeftSideRow); + if (relationship.RighSideRow != null) loadBearing.Add(relationship.RighSideRow); + } + + var withoutArtefacts = new List(); + + foreach (var table in tables.Where(t => t.Type == TableType.InSolution)) + { + if (!references.HasOwn(table.LogicalName)) + { + withoutArtefacts.Add(table.LogicalName); + } + + foreach (var row in table.Rows.ToList()) + { + // A primary key and a state model describe the table whatever refers to + // them, and an edge's own column cannot go without crashing a translator. + if (row.RowType is RowType.Primarykey or RowType.State or RowType.Status) continue; + if (loadBearing.Contains(row)) continue; + + var reason = ReasonToDrop(table, row, references, scope.AuthorPrefixes); + if (reason == null) continue; + + table.Rows.Remove(row); + scope.DroppedColumns.Add(new DroppedColumn(table.LogicalName, row.Name, reason.Value)); + } + } + + _logger.LogInformation( + "Narrowed {Tables} table(s) to the columns their own forms, views, workflows, sitemaps and .cs/.ts sources refer to; " + + "dropped {Dropped}. A dropped column is one no reference was found for, which is not the same as one that is unused.", + tables.Count(t => t.Type == TableType.InSolution), scope.DroppedColumns.Count); + + if (withoutArtefacts.Count > 0) + { + // Nothing referenced these tables' columns because nothing could: they have no + // forms or views of their own, so only their keys and relationships survive. + _logger.LogWarning( + "{Count} table(s) have no forms, views or charts of their own, so only their keys and relationships remain: {Tables}.", + withoutArtefacts.Count, string.Join(", ", withoutArtefacts.OrderBy(x => x, StringComparer.Ordinal))); + } + } + + private static DropReason? ReasonToDrop(Table table, TableRow row, ReferenceIndex references, HashSet authorPrefixes) + { + // Checked first, so a column dropped as plumbing is not reported as unreferenced. + if (IsPlatformPlumbing(table, row)) return DropReason.PlatformPlumbing; + + if (references.OwnedBy(table.LogicalName, row.Name)) return null; + + // An artefact belonging to no single table — an app module, a sitemap, a plug-in — + // can only be matched by name, and a platform column's name is the same on every + // table in the org. Letting one rescue createdon puts it back on all nineteen + // tables, which is the defect this filter exists to remove. An author's column is + // named once, so the same evidence is worth trusting there: measured at 30 columns + // across two real apps. + if (!IsPlatformColumn(row, authorPrefixes) && references.Unattributed(row.Name)) return null; + + return DropReason.NoReferenceFound; + } + + /// + /// Platform plumbing a reader of the model never needs, even where something refers to it. + /// + private static bool IsPlatformPlumbing(Table table, TableRow row) + => row.IsLogical == true + || ProcessFlowColumns.Contains(row.Name, StringComparer.OrdinalIgnoreCase) + || IsBaseCurrencyTwin(table, row); + + /// + /// The shadow the platform maintains in the base currency beside an author's money + /// column. Nothing in the metadata separates the two, so this is a name pairing. + /// + private static bool IsBaseCurrencyTwin(Table table, TableRow row) + => row.RowType == RowType.Money + && row.Name.EndsWith(BaseCurrencySuffix, StringComparison.OrdinalIgnoreCase) + && table.Rows.Any(other => other.RowType == RowType.Money + && string.Equals(other.Name, row.Name[..^BaseCurrencySuffix.Length], StringComparison.OrdinalIgnoreCase)); + + /// + /// Whether the platform created a column rather than an author. Dataverse gives an + /// author's column the publisher's prefix and its own columns none, which is the whole + /// discriminator. With no prefixes to check against, nothing is called the platform's + /// rather than narrowing the output on no evidence. + /// + private static bool IsPlatformColumn(TableRow row, HashSet authorPrefixes) + => authorPrefixes.Count > 0 + && !authorPrefixes.Any(prefix => row.Name.StartsWith(prefix + "_", StringComparison.OrdinalIgnoreCase)); + + /// + /// Every token in every referencing artefact, credited to the table whose folder it sits + /// under. An artefact outside one — an app module, a sitemap, a plug-in source — is + /// credited to no table and matched by name alone. + /// + private static ReferenceIndex CollectReferences(IEnumerable searchRoots) + { + var index = new ReferenceIndex(); + + foreach (var root in searchRoots.Where(Directory.Exists).Select(Path.GetFullPath).Distinct()) + { + foreach (var file in Directory.EnumerateFiles(root, "*", SearchOption.AllDirectories)) + { + if (!ShouldScan(file)) continue; + + var owner = OwnerFromPath(file); + + try + { + foreach (Match match in TokenPattern.Matches(File.ReadAllText(file))) + { + index.Add(owner, match.Value); + } + } + catch (IOException ex) + { + _logger.LogWarning(ex, "Could not read {File} while looking for column references.", file); + } + } + } + + return index; + } + + /// The segment naming the entity whose folder this artefact lives under, or null + /// for one that sits outside any and therefore speaks for many tables. + private static string? OwnerFromPath(string file) + { + var segments = (Path.GetDirectoryName(file) ?? string.Empty) + .Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + + for (var i = 0; i < segments.Length - 1; i++) + { + if (string.Equals(segments[i], "Entities", StringComparison.OrdinalIgnoreCase)) + { + return segments[i + 1]; + } + } + + return null; + } + + private static bool ShouldScan(string file) + { + // Plug-in and script sources sit outside the declarations, so a reference from one + // is invisible without reading them: measured at 14 columns across two real apps. + if (CodeExtensions.Contains(Path.GetExtension(file), StringComparer.OrdinalIgnoreCase)) + { + return true; + } + + // Entity.xml declares columns rather than referencing them. + if (string.Equals(Path.GetFileName(file), "Entity.xml", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + var directory = Path.GetDirectoryName(file) ?? string.Empty; + + return ReferencingFolders.Any(folder => + directory.Contains(Path.DirectorySeparatorChar + folder + Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase) + || directory.EndsWith(Path.DirectorySeparatorChar + folder, StringComparison.OrdinalIgnoreCase)); + } + + /// Which columns each table's own artefacts refer to, plus the references that + /// belong to no single table and therefore count for all of them. + private sealed class ReferenceIndex + { + private readonly Dictionary> _byTable = new(StringComparer.OrdinalIgnoreCase); + private readonly HashSet _unattributed = new(StringComparer.OrdinalIgnoreCase); + + public void Add(string? table, string token) + { + if (table == null) + { + _unattributed.Add(token); + return; + } + + if (!_byTable.TryGetValue(table, out var tokens)) + { + _byTable[table] = tokens = new HashSet(StringComparer.OrdinalIgnoreCase); + } + + tokens.Add(token); + } + + public bool HasOwn(string table) => _byTable.ContainsKey(table); + + /// An artefact of this table names this column. + public bool OwnedBy(string table, string column) + => _byTable.TryGetValue(table, out var tokens) && tokens.Contains(column); + + /// Something names this column, but nothing says which table it meant. + public bool Unattributed(string column) => _unattributed.Contains(column); + } +} diff --git a/src/TALXIS.CLI.Features.Data/DataModelConverter/AppScope/DroppedColumn.cs b/src/TALXIS.CLI.Features.Data/DataModelConverter/AppScope/DroppedColumn.cs new file mode 100644 index 00000000..145603f8 --- /dev/null +++ b/src/TALXIS.CLI.Features.Data/DataModelConverter/AppScope/DroppedColumn.cs @@ -0,0 +1,21 @@ +namespace TALXIS.CLI.Features.Data.DataModelConverter.AppScope; + +/// Why a column was left out, so a reader can tell a judgement from an absence. +public enum DropReason +{ + /// Nothing belonging to the column's own table referred to it. + NoReferenceFound = 0, + + /// Platform plumbing a reader never needs, even where something refers + /// to it. + PlatformPlumbing = 1 +} + +/// One column left out of the conversion. +/// Logical name of the table the column was declared on. +/// The column's own name. +/// Why it was left out. +public sealed record DroppedColumn(string Table, string Column, DropReason Reason) +{ + public override string ToString() => $"{Table}.{Column}"; +} diff --git a/src/TALXIS.CLI.Features.Data/DataModelConverter/DataModelConverterService.cs b/src/TALXIS.CLI.Features.Data/DataModelConverter/DataModelConverterService.cs index 36a1d17c..b40a9d82 100644 --- a/src/TALXIS.CLI.Features.Data/DataModelConverter/DataModelConverterService.cs +++ b/src/TALXIS.CLI.Features.Data/DataModelConverter/DataModelConverterService.cs @@ -13,12 +13,14 @@ using TALXIS.CLI.Features.Data.DataModelConverter.Translators; using TALXIS.CLI.Logging; +using TALXIS.CLI.Features.Data.DataModelConverter.AppScope; + namespace TALXIS.CLI.Features.Data.DataModelConverter; public class DataModelConverterService { private static readonly ILogger _logger = TxcLoggerFactory.CreateLogger(nameof(DataModelConverterService)); - private static readonly string[] SupportedFormats = ["dbml", "sql", "edmx", "ribbon"]; + private static readonly string[] SupportedFormats = ["dbml", "sql", "plainsql", "edmx", "ribbon"]; /// /// Parses a Power Platform solution from a solution project folder, a declarations @@ -35,29 +37,75 @@ public class DataModelConverterService /// /// public static void ConvertModel(string inputPath, string targetFormat, string outputFilePath) + => ConvertModel([inputPath], targetFormat, outputFilePath); + + /// + /// Converts one or more inputs into a single model. Each input is resolved + /// independently -- a solution project folder, a declarations folder, or a .zip -- and + /// they may be mixed. Earlier inputs take precedence where two declare the same + /// attribute differently. + /// + public static void ConvertModel(List inputPaths, string targetFormat, string outputFilePath) + => ConvertModel(inputPaths, targetFormat, outputFilePath, null, null); + + /// + /// Converts one or more inputs into a single model, optionally narrowed to the tables a + /// model-driven app is built on. is where app modules + /// are looked for; apps and entity schema live in different modules, so this is usually + /// a repository root rather than a declarations folder. + /// + public static void ConvertModel(List inputPaths, string targetFormat, string outputFilePath, string? appUniqueName, List? appSearchRoots) + => ConvertModel(inputPaths, targetFormat, outputFilePath, appUniqueName, appSearchRoots, DetailLevel.Full); + + /// + /// of narrows each table to + /// the columns its own artefacts refer to, and drops the platform plumbing a reader + /// never needs. + /// + public static IReadOnlyList ConvertModel(List inputPaths, string targetFormat, string outputFilePath, string? appUniqueName, List? appSearchRoots, DetailLevel detail) { if (!SupportedFormats.Contains(targetFormat.ToLower())) throw new ArgumentException($"Unsupported target format '{targetFormat}'. Supported formats are: {string.Join(", ", SupportedFormats)}."); - ParsedModel parsedModel; + if (inputPaths is null || inputPaths.Count == 0) + throw new ArgumentException("At least one input path is required."); - if (Directory.Exists(inputPath)) - { - var declarationsPath = ResolveDeclarationsFolder(inputPath); - parsedModel = ParseModelFolder(declarationsPath); - } - else if (File.Exists(inputPath)) + List modules = []; + foreach (var inputPath in inputPaths) { - using var fileStream = new FileStream(inputPath, FileMode.Open, FileAccess.Read); - using var memoryStream = new MemoryStream(); - fileStream.CopyTo(memoryStream); - parsedModel = ParseModel(Convert.ToBase64String(memoryStream.ToArray())); + if (Directory.Exists(inputPath)) + { + modules.Add(ParseFolderIntoModule(ResolveDeclarationsFolder(inputPath))); + } + else if (File.Exists(inputPath)) + { + using var fileStream = new FileStream(inputPath, FileMode.Open, FileAccess.Read); + using var memoryStream = new MemoryStream(); + fileStream.CopyTo(memoryStream); + modules.Add(ParseZipIntoModule(Convert.ToBase64String(memoryStream.ToArray()))); + } + else + { + throw new FileNotFoundException($"Input path '{inputPath}' does not exist."); + } } - else + + ResolvedAppScope? appScope = null; + if (!string.IsNullOrWhiteSpace(appUniqueName)) { - throw new FileNotFoundException($"Input path '{inputPath}' does not exist."); + var roots = appSearchRoots is { Count: > 0 } ? appSearchRoots : inputPaths; + appScope = AppScopeResolver.Resolve(roots, appUniqueName); + appScope.Detail = detail; + appScope.SearchRoots = [.. roots]; + + foreach (var prefix in modules.Select(m => m.CustomizationPrefix).Where(p => !string.IsNullOrWhiteSpace(p))) + { + appScope.AuthorPrefixes.Add(prefix!); + } } + var parsedModel = ParseModules(modules, appScope); + var resultString = targetFormat.ToLower() switch { "edmx" => ConvertToEDMX(parsedModel), @@ -67,8 +115,12 @@ public static void ConvertModel(string inputPath, string targetFormat, string ou _ => ConvertToDBML(parsedModel) }; - using var writer = new StreamWriter(outputFilePath); - writer.Write(resultString); + using (var writer = new StreamWriter(outputFilePath)) + { + writer.Write(resultString); + } + + return appScope?.DroppedColumns ?? []; } /// @@ -255,11 +307,72 @@ public static string ConvertToEDMX(ParsedModel model) } public static ParsedModel ParseModelFolder(string folderPath) + => ParseModelFolders([folderPath]); + + /// + /// Parses several declarations folders into one model, merging attribute-level. + /// A project's model is rarely one solution: the base product ships several modules + /// that each declare part of a shared table, so converting them separately and + /// concatenating the files loses everything but the first declaration of each table. + /// + public static ParsedModel ParseModelFolders(List folderPaths) + => ParseModules([.. folderPaths.Select(ParseFolderIntoModule)]); + + /// + /// Names a module after the folders that own its declarations, so tables can be + /// attributed once several inputs are merged. Several segments are kept because the + /// leaf is almost always "Model" -- one segment would give every input the same name + /// and, with the colour derived from it, the same colour. + /// + private static string ModuleNameFor(string declarationsFolder) + { + var full = Path.GetFullPath(declarationsFolder) + .TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + var segments = full.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) + .Where(x => x.Length > 0) + .ToList(); + + // Drop the trailing "Declarations" (or "CDS") folder; it carries no information. + if (segments.Count > 1 && (segments[^1].Equals("Declarations", StringComparison.OrdinalIgnoreCase) + || segments[^1].Equals("CDS", StringComparison.OrdinalIgnoreCase))) + { + segments.RemoveAt(segments.Count - 1); + } + + return string.Join('/', segments.TakeLast(3)); + } + + /// + /// Finds every declarations folder beneath a root, by looking for the entity + /// declarations themselves rather than for a folder name -- modules keep them under + /// "Declarations" or, in older ones, "CDS". + /// + public static List DiscoverDeclarationFolders(string root) + { + if (!Directory.Exists(root)) throw new DirectoryNotFoundException($"Root '{root}' does not exist."); + + return [.. Directory.EnumerateFiles(root, "Entity.xml", SearchOption.AllDirectories) + .Select(f => Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(f)))) + .Where(d => !string.IsNullOrEmpty(d)) + .Select(d => Path.GetFullPath(d!)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(d => d, StringComparer.OrdinalIgnoreCase)]; + } + + private static Module ParseFolderIntoModule(string folderPath) { - Module module = new(); + Module module = new() + { + ModuleName = ModuleNameFor(folderPath), + CustomizationPrefix = PrefixFromFolder(folderPath) + }; // Get files named Entity.xml in subfolders - var entityFiles = Directory.GetFiles(folderPath, "Entity.xml", SearchOption.AllDirectories); + // Ordered: Directory.GetFiles gives no ordering guarantee, so without this the + // table, relationship and enum order in the output varies by filesystem and the + // result cannot be committed or diffed. + var entityFiles = Directory.GetFiles(folderPath, "Entity.xml", SearchOption.AllDirectories) + .OrderBy(f => f, StringComparer.Ordinal).ToArray(); foreach (var file in entityFiles) { @@ -284,7 +397,7 @@ public static ParsedModel ParseModelFolder(string folderPath) // Get files in folder Other/Relationships (directory may not exist in scaffolded solutions) var relationshipsDir = Path.Combine(folderPath, "Other", "Relationships"); var relationshipFiles = Directory.Exists(relationshipsDir) - ? Directory.GetFiles(relationshipsDir, "*.xml", SearchOption.AllDirectories) + ? [.. Directory.GetFiles(relationshipsDir, "*.xml", SearchOption.AllDirectories).OrderBy(f => f, StringComparer.Ordinal)] : Array.Empty(); foreach (var file in relationshipFiles) { @@ -302,7 +415,7 @@ public static ParsedModel ParseModelFolder(string folderPath) // Get files in folder called OptionSets (directory may not exist in scaffolded solutions) var optionsetsDir = Path.Combine(folderPath, "OptionSets"); var optionsetFiles = Directory.Exists(optionsetsDir) - ? Directory.GetFiles(optionsetsDir, "*.xml", SearchOption.AllDirectories) + ? [.. Directory.GetFiles(optionsetsDir, "*.xml", SearchOption.AllDirectories).OrderBy(f => f, StringComparer.Ordinal)] : Array.Empty(); foreach (var file in optionsetFiles) { @@ -317,8 +430,52 @@ public static ParsedModel ParseModelFolder(string folderPath) } } - return ParseModules([module]); + return module; + } + + private static Module ParseZipIntoModule(string base64solution) + { + using ZipArchive archive = new(new MemoryStream(Convert.FromBase64String(base64solution))); + + var customizationsxml = archive.Entries.FirstOrDefault(x => x.FullName.Equals("customizations.xml", StringComparison.OrdinalIgnoreCase)); + var solutionxml = archive.Entries.FirstOrDefault(x => x.FullName.Equals("solution.xml", StringComparison.OrdinalIgnoreCase)); + + if (customizationsxml == null || solutionxml == null) + { + throw new FileNotFoundException("The solution archive does not contain the required customizations.xml or solution.xml files."); + } + + var manifest = XDocument.Load(solutionxml.Open()); + return new Module( + manifest.Descendants().First(x => x.Name == "UniqueName").Value, + XDocument.Load(customizationsxml.Open())) + { + CustomizationPrefix = Module.PrefixFrom(manifest) + }; + } + + /// + /// A declarations folder keeps its manifest at Other/Solution.xml, beside the + /// relationships this converter already reads. + /// + private static string? PrefixFromFolder(string folderPath) + { + var manifestPath = Path.Combine(folderPath, "Other", "Solution.xml"); + if (!File.Exists(manifestPath)) + { + return null; + } + + try + { + return Module.PrefixFrom(XDocument.Load(manifestPath)); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Could not read the publisher prefix from {File}", manifestPath); + return null; + } } public static ParsedModel ParseModel(string? base64solution) @@ -338,39 +495,35 @@ public static ParsedModel ParseModel(List base64solution) foreach (var solution in base64solution) { - using ZipArchive archive = new(new MemoryStream(Convert.FromBase64String(solution))); - - var customizationsxml = archive.Entries.FirstOrDefault(x => x.FullName.Equals("customizations.xml", StringComparison.OrdinalIgnoreCase)); - var solutionxml = archive.Entries.FirstOrDefault(x => x.FullName.Equals("solution.xml", StringComparison.OrdinalIgnoreCase)); - - if (customizationsxml == null || solutionxml == null) - { - throw new FileNotFoundException("The solution archive does not contain the required customizations.xml or solution.xml files."); - } - - Module foundModule = new(XDocument.Load(solutionxml.Open()).Descendants().First(x => x.Name == "UniqueName").Value, XDocument.Load(customizationsxml.Open())); - - modules.Add(foundModule); + modules.Add(ParseZipIntoModule(solution)); } return ParseModules(modules); } public static ParsedModel ParseModules(List modules) + => ParseModules(modules, null); + + public static ParsedModel ParseModules(List modules, ResolvedAppScope? appScope) { List
EntityTables = ParseEntities(modules); List EntityOptionSets = ParseOptionSets(modules); - // Remove optionset rows without optionsets defined + // Downgrade optionset rows whose optionset is not resolvable here (declared with no + // options, owned by another module, or platform-owned) rather than dropping the column. var validOptionSetNames = EntityOptionSets.Select(x => x.LocalizedName).ToHashSet(StringComparer.OrdinalIgnoreCase); - foreach (var entity in EntityTables) + foreach (var row in EntityTables + .SelectMany(entity => entity.Rows) + .Where(row => + row.RowType is (RowType.Picklist or RowType.Multiselectoptionset or RowType.State or RowType.Status or RowType.Bit) + && !validOptionSetNames.Contains(row.OptionSetName))) { - entity.Rows = [.. entity.Rows - .Where(row => - row.RowType is not (RowType.Picklist or RowType.Multiselectoptionset or RowType.State or RowType.Status or RowType.Bit) - || validOptionSetNames.Contains(row.OptionSetName) - )]; + // Clearing OptionSetName is enough and is all that is needed: it is what + // ToDbDiagramNotation prefers over RowType, so leaving it set would make the + // column reference an Enum that was never emitted. RowType is deliberately + // left alone so each translator keeps its own handling for the kind. + row.OptionSetName = string.Empty; } // Fill in setnames where missing with placeholder logical names @@ -379,7 +532,34 @@ row.RowType is not (RowType.Picklist or RowType.Multiselectoptionset or RowType. entity.SetName = entity.LogicalName; } - List EntityRelationships = ParseRelationships(modules, EntityTables); + // Before relationships: a table dropped here must not reappear as a stub created + // for a relationship that pointed at it. + if (appScope != null) + { + AppScopeFilter.ApplyTableScope(EntityTables, appScope); + } + + List EntityRelationships = ParseRelationships(modules, EntityTables, appScope); + + if (appScope is { Detail: DetailLevel.Minimal }) + { + // After relationships: the set of columns an edge depends on is only knowable + // once they exist. + AttributeReferenceFilter.Apply(EntityTables, EntityRelationships, appScope); + } + + if (appScope != null) + { + // Option sets belonging to tables the scope removed would otherwise still be + // emitted, leaving more enum declarations in the output than columns using them. + var referenced = EntityTables + .SelectMany(t => t.Rows) + .Select(r => r.OptionSetName) + .Where(n => !string.IsNullOrEmpty(n)) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + EntityOptionSets.RemoveAll(o => !referenced.Contains(o.LocalizedName)); + } return new ParsedModel() { @@ -391,6 +571,38 @@ row.RowType is not (RowType.Picklist or RowType.Multiselectoptionset or RowType. } public static List ParseRelationships(List modules, List
EntityTables) + => ParseRelationships(modules, EntityTables, null); + + /// + /// Which kind of stub stands in for a table an edge points at. One the inputs do declare + /// was removed by app scoping; without app scoping every stub is genuinely absent. + /// + private static TableType StubTypeFor(ResolvedAppScope? appScope, string logicalName) + => appScope?.AllDeclaredTableLogicalNames.Contains(logicalName) == true + ? TableType.NotInApp + : TableType.NotInSolution; + + private static bool IsInAppScope(XElement relationship, ResolvedAppScope appScope) + { + if (relationship.Element("EntityRelationshipType")?.Value == "ManyToMany") + { + var firstInScope = appScope.TableLogicalNames.Contains(relationship.Element("FirstEntityName")?.Value ?? string.Empty); + var secondInScope = appScope.TableLogicalNames.Contains(relationship.Element("SecondEntityName")?.Value ?? string.Empty); + + // An N:N is part of an app's own design only when both its tables are. Admitting + // it on one side drags a shared table's whole association network in: systemuser + // alone contributed nine intersects and four far-side stubs to one real app. + // This gate is read before the branch that builds those tables, so tightening it + // withholds the tables and the edge together rather than leaving a dangling end. + return appScope.Detail == DetailLevel.Minimal + ? firstInScope && secondInScope + : firstInScope || secondInScope; + } + + return appScope.TableLogicalNames.Contains(relationship.Element("ReferencingEntityName")?.Value ?? string.Empty); + } + + public static List ParseRelationships(List modules, List
EntityTables, ResolvedAppScope? appScope) { List EntityRelationships = new(); @@ -401,25 +613,46 @@ public static List ParseRelationships(List modules, List ParseRelationships(List modules, List x.RowType == RowType.Primarykey), connectionTable, - connectionTable.Rows.FirstOrDefault(x => x.Name == firstEntityTable.LogicalName + "id")); + connectionTable.Rows.FirstOrDefault(x => x.Name == firstRowName)); - var secondToMid = new Relationship(relationship.Attribute("Name").Value, + var secondToMid = new Relationship(secondRelationshipName, "ManyToOne", secondEntityTable, secondEntityTable.Rows.FirstOrDefault(x => x.RowType == RowType.Primarykey), connectionTable, - connectionTable.Rows.FirstOrDefault(x => x.Name == secondEntityTable.LogicalName + "id")); + connectionTable.Rows.FirstOrDefault(x => x.Name == secondRowName)); EntityRelationships.Add(firstToMid); EntityRelationships.Add(secondToMid); @@ -463,7 +705,7 @@ public static List ParseRelationships(List modules, List ParseRelationships(List modules, List ParseRelationships(List modules, List x.RowType == RowType.Primarykey)); - if (EntityRelationships.FirstOrDefault(x => x.LeftSideTable == entityRelationship.LeftSideTable && x.RighSideTable == entityRelationship.RighSideTable) == default) + if (EntityRelationships.FirstOrDefault(x => x.LeftSideTable == entityRelationship.LeftSideTable && x.LeftSideRow == entityRelationship.LeftSideRow && x.RighSideTable == entityRelationship.RighSideTable) == default) { EntityRelationships.Add(entityRelationship); } diff --git a/src/TALXIS.CLI.Features.Data/DataModelConverter/DetailLevel.cs b/src/TALXIS.CLI.Features.Data/DataModelConverter/DetailLevel.cs new file mode 100644 index 00000000..46ab485c --- /dev/null +++ b/src/TALXIS.CLI.Features.Data/DataModelConverter/DetailLevel.cs @@ -0,0 +1,14 @@ +namespace TALXIS.CLI.Features.Data.DataModelConverter; + +/// How much of a solution's metadata the conversion emits. +public enum DetailLevel +{ + /// Everything the inputs declare. + Full = 0, + + /// + /// Only what shows how the app was built: tables the app is built on, and the + /// columns something belonging to those tables refers to. Not a schema export. + /// + Minimal = 1 +} diff --git a/src/TALXIS.CLI.Features.Data/DataModelConverter/Model/Module.cs b/src/TALXIS.CLI.Features.Data/DataModelConverter/Model/Module.cs index 4a8ea5d7..3ed4cb17 100644 --- a/src/TALXIS.CLI.Features.Data/DataModelConverter/Model/Module.cs +++ b/src/TALXIS.CLI.Features.Data/DataModelConverter/Model/Module.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Security.Cryptography; using System.Text; using System.Threading.Tasks; using System.Xml.Linq; @@ -9,31 +10,46 @@ namespace TALXIS.CLI.Features.Data.DataModelConverter.Model; public class Module { - public Module() - { - var random = new Random(); - Colorhex = string.Format("#{0:X6}", random.Next(0x1000000)); - } + public Module() { } public Module(string module, XDocument xml) { ModuleName = module; XmlDoc = xml; - var random = new Random(); - Colorhex = string.Format("#{0:X6}", random.Next(0x1000000)); - entities = XmlDoc.Descendants().Where(x => x.Name == "Entity").ToList(); relationships = XmlDoc.Descendants().Where(x => x.Name == "EntityRelationship").ToList(); optionsets = XmlDoc.Descendants().Where(x => x.Name == "optionset").ToList(); } + /// Reads the publisher prefix out of a solution manifest, from either a folder's + /// Other/Solution.xml or an archive's solution.xml. + public static string? PrefixFrom(XDocument solutionManifest) => + solutionManifest.Descendants().FirstOrDefault(x => x.Name == "CustomizationPrefix")?.Value; + public string ModuleName { get; set; } = ""; public XDocument XmlDoc { get; set; } = new XDocument(); + /// The publisher prefix this module's own columns carry, from its Solution.xml. + /// Ground truth for telling an author's column from a platform one, which the + /// per-attribute metadata alone gets wrong on primary keys and name fields. + public string? CustomizationPrefix { get; set; } + public List entities = []; public List relationships = []; public List optionsets = []; - public string Colorhex { get; } + /// Computed, not assigned in the constructor: an object initializer sets + /// ModuleName after the constructor body runs, which would colour every module + /// from an empty name. + public string Colorhex => ColourFor(ModuleName); + + /// + /// Derives the module colour from its name so the same input always converts to the + /// same bytes. A random colour made every conversion a spurious diff, which meant a + /// generated diagram could not be committed or compared across a model change. + /// Not string.GetHashCode, which is randomised per process on .NET Core. + /// + private static string ColourFor(string moduleName) + => "#" + Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(moduleName ?? string.Empty)))[..6]; } diff --git a/src/TALXIS.CLI.Features.Data/DataModelConverter/Model/Table.cs b/src/TALXIS.CLI.Features.Data/DataModelConverter/Model/Table.cs index 5ccd5960..337d0034 100644 --- a/src/TALXIS.CLI.Features.Data/DataModelConverter/Model/Table.cs +++ b/src/TALXIS.CLI.Features.Data/DataModelConverter/Model/Table.cs @@ -1,4 +1,6 @@ -using DocumentFormat.OpenXml.Vml.Office; +using TALXIS.CLI.Logging; +using Microsoft.Extensions.Logging; +using DocumentFormat.OpenXml.Vml.Office; using System.Text.Json.Serialization; using System; using System.Collections.Generic; @@ -15,11 +17,20 @@ public enum TableType { InSolution = 0, NotInSolution = 1, - ConnectionTable = 2 + ConnectionTable = 2, + + /// + /// A stub for a table an input does declare, which is only a stub because app scoping + /// dropped it. Without this a diagram marks most of its stubs as missing from the + /// solution, which is untrue: 13 of 14 in one real app were declared in the same inputs. + /// + NotInApp = 3 } public class Table { + private static readonly ILogger _logger = TxcLoggerFactory.CreateLogger(nameof(Table)); + public Table() { } public Table(XElement element) @@ -58,8 +69,34 @@ public void ParseMultipleRowsFromXml(List xElements) foreach (var element in xElements) { var row = TableRow.ParseXElement(element); - if (row != null) + if (row == null) + continue; + + var existing = Rows.FirstOrDefault(x => string.Equals(x.Name, row.Name, StringComparison.OrdinalIgnoreCase)); + if (existing == null) + { Rows.Add(row); + continue; + } + + // First non-null wins rather than first input: export styles differ in whether + // they emit this at all, so a later, more complete declaration still counts. + existing.IsLogical ??= row.IsLogical; + + if (existing.RowType != row.RowType) + { + // Several modules extending one shared table is the normal case for a + // layered product, so a divergent declaration warns rather than aborting. + // First input wins, which makes the result deterministic in input order. + _logger.LogWarning( + "Attribute {Table}.{Attribute} is declared as {ExistingType} in one input and {NewType} in another; keeping the first.", + LogicalName, row.Name, existing.RowType, row.RowType); + } + else if (row.MaxLenght > existing.MaxLenght) + { + // Widen, never narrow: a consumer breaks on too little room, not too much. + existing.MaxLenght = row.MaxLenght; + } } } diff --git a/src/TALXIS.CLI.Features.Data/DataModelConverter/Model/TableRow.cs b/src/TALXIS.CLI.Features.Data/DataModelConverter/Model/TableRow.cs index 70aa4f3a..f3007bde 100644 --- a/src/TALXIS.CLI.Features.Data/DataModelConverter/Model/TableRow.cs +++ b/src/TALXIS.CLI.Features.Data/DataModelConverter/Model/TableRow.cs @@ -25,6 +25,11 @@ public TableRow(string name, RowType rowType) public string OptionSetName { get; set; } public RowType RowType { get; set; } + /// Whether the platform computes this column rather than storing it, from the + /// declaration's own IsLogical. OwningUser and OwningTeam are the + /// common cases. + public bool? IsLogical { get; set; } + internal static TableRow? ParseXElement(XElement attribute) { string optionsetName = string.Empty; @@ -115,7 +120,8 @@ public TableRow(string name, RowType rowType) return new TableRow(attribute.Attribute("PhysicalName").Value.ToLower(), rowType) { MaxLenght = maxLength, - OptionSetName = optionsetName + OptionSetName = optionsetName, + IsLogical = attribute.Element("IsLogical") is { } isLogical ? isLogical.Value == "1" : null }; } diff --git a/src/TALXIS.CLI.Features.Data/DataModelConverter/Translators/DBDiagramTranslator.cs b/src/TALXIS.CLI.Features.Data/DataModelConverter/Translators/DBDiagramTranslator.cs index 8b7f4382..2340b966 100644 --- a/src/TALXIS.CLI.Features.Data/DataModelConverter/Translators/DBDiagramTranslator.cs +++ b/src/TALXIS.CLI.Features.Data/DataModelConverter/Translators/DBDiagramTranslator.cs @@ -27,6 +27,11 @@ public static string ToDbDiagramNotation(this Table table) case TableType.NotInSolution: result += "[headercolor: #c0392b] "; break; + case TableType.NotInApp: + // Grey, not the red of a table nothing declares: this one is in the + // solution, just not in this app, which is a different fact for the reader. + result += "[headercolor: #7f8c8d] //declared outside this app \n"; + break; case TableType.ConnectionTable: result += "[headercolor: #27ae60] "; break; diff --git a/tests/TALXIS.CLI.Tests/Data/DataModelConverter/AppScopeTests.cs b/tests/TALXIS.CLI.Tests/Data/DataModelConverter/AppScopeTests.cs new file mode 100644 index 00000000..80ef199e --- /dev/null +++ b/tests/TALXIS.CLI.Tests/Data/DataModelConverter/AppScopeTests.cs @@ -0,0 +1,214 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml.Linq; +using TALXIS.CLI.Features.Data.DataModelConverter; +using TALXIS.CLI.Features.Data.DataModelConverter.AppScope; +using Model = TALXIS.CLI.Features.Data.DataModelConverter.Model; +using Xunit; + +namespace TALXIS.CLI.Tests.Data.DataModelConverter; + +/// +/// Resolving which tables a model-driven app is built on, from source alone. The shapes +/// exercised here are the ones real repositories actually contain: declarations under +/// "CDS" as well as "Declarations", one app declared across several files, and a folder +/// whose name differs in case from the UniqueName inside it. +/// +public class AppScopeTests : IDisposable +{ + private readonly string _root = Path.Combine(Path.GetTempPath(), "txc-app-" + Path.GetRandomFileName()); + + public void Dispose() + { + if (Directory.Exists(_root)) Directory.Delete(_root, recursive: true); + GC.SuppressFinalize(this); + } + + private string WriteAppModule(string folderName, string uniqueName, string componentsXml, string declarationsFolder = "Declarations") + { + var dir = Path.Combine(_root, "module", declarationsFolder, "AppModules", folderName); + Directory.CreateDirectory(dir); + var file = Path.Combine(dir, "AppModule_managed.xml"); + File.WriteAllText(file, $""" + + {uniqueName} + + {componentsXml} + + + """); + return file; + } + + private static string Component(string type, string schemaName) => + $""""""; + + // ---- resolution ------------------------------------------------------------------ + + [Fact] + public void OnlyEntityComponentsContributeTables() + { + WriteAppModule("contoso_app", "contoso_app", + Component("1", "contoso_thing") + + Component("26", "some_view") // saved query + + Component("60", "some_form") // system form + + Component("62", "contoso_app") // the app's own sitemap + + Component("1", "account")); + + var scope = AppScopeResolver.Resolve([_root], "contoso_app"); + + Assert.Equal(new[] { "account", "contoso_thing" }, scope.TableLogicalNames.OrderBy(x => x)); + } + + [Fact] + public void DeclarationsUnderCdsFolder_AreStillFound() + { + // Older modules keep their declarations under "CDS" rather than "Declarations"; + // a search anchored on either folder name misses the other. + WriteAppModule("contoso_app", "contoso_app", Component("1", "contoso_thing"), declarationsFolder: "CDS"); + + var scope = AppScopeResolver.Resolve([_root], "contoso_app"); + + Assert.Contains("contoso_thing", scope.TableLogicalNames); + } + + [Fact] + public void OneAppDeclaredAcrossSeveralFiles_UnionsItsComponents() + { + // A second area can contribute components to an app it does not own; the app's real + // component set is the union of every file that declares it. + WriteAppModule("contoso_app", "contoso_app", Component("1", "contoso_first")); + var second = Path.Combine(_root, "other", "Declarations", "AppModules", "contoso_app"); + Directory.CreateDirectory(second); + File.WriteAllText(Path.Combine(second, "AppModule_managed.xml"), """ + + contoso_app + + + + + """); + + var scope = AppScopeResolver.Resolve([_root], "contoso_app"); + + Assert.Contains("contoso_first", scope.TableLogicalNames); + Assert.Contains("contoso_second", scope.TableLogicalNames); + Assert.Equal(2, scope.SourceFiles.Count); + } + + [Fact] + public void IdentityComesFromFileContent_NotTheFolderName() + { + // The folder is only a locator, and its casing can differ from the declared name. + WriteAppModule("Contoso_App", "contoso_app", Component("1", "contoso_thing")); + + var scope = AppScopeResolver.Resolve([_root], "contoso_app"); + + Assert.Contains("contoso_thing", scope.TableLogicalNames); + } + + [Fact] + public void SiteMapEntities_AreIncluded_FromBothTheAttributeAndTheUrl() + { + WriteAppModule("contoso_app", "contoso_app", Component("1", "contoso_thing")); + var dir = Path.Combine(_root, "module", "Declarations", "AppModuleSiteMaps", "contoso_app"); + Directory.CreateDirectory(dir); + File.WriteAllText(Path.Combine(dir, "AppModuleSiteMap_managed.xml"), """ + + contoso_app + + + + """); + + var scope = AppScopeResolver.Resolve([_root], "contoso_app"); + + Assert.Contains("contoso_viaattribute", scope.TableLogicalNames); + Assert.Contains("contoso_viaurl", scope.TableLogicalNames); + } + + [Fact] + public void UnknownApp_FailsWithTheNamesItDidFind() + { + WriteAppModule("contoso_app", "contoso_app", Component("1", "contoso_thing")); + + var ex = Assert.Throws(() => AppScopeResolver.Resolve([_root], "contoso_typo")); + + Assert.Contains("contoso_app", ex.Message); + } + + // ---- filtering ------------------------------------------------------------------- + + private static XElement Entity(string logicalName) => + XElement.Parse($""" + + {logicalName} + + primarykey + lookup + + + """); + + private static XElement OneToMany(string child, string attr, string parent) => + XElement.Parse($""" + + OneToMany + {child} + {parent} + {attr} + + """); + + [Fact] + public void TablesOutsideTheApp_AreDropped_AndDoNotReturnAsRelationshipStubs() + { + // The ordering that matters: filtering after relationships were built would let a + // relationship among the dropped tables synthesise them straight back as stubs. + var module = new Model.Module { ModuleName = "test" }; + module.entities.AddRange([Entity("contoso_inapp"), Entity("contoso_elsewhere"), Entity("contoso_alsoelsewhere")]); + module.relationships.Add(OneToMany("contoso_elsewhere", "contoso_elsewhere_lookup", "contoso_alsoelsewhere")); + + var scope = new ResolvedAppScope { UniqueName = "contoso_app" }; + scope.TableLogicalNames.Add("contoso_inapp"); + + var model = DataModelConverterService.ParseModules([module], scope); + + Assert.Contains(model.tables, t => t.LogicalName == "contoso_inapp"); + Assert.DoesNotContain(model.tables, t => t.LogicalName == "contoso_elsewhere"); + Assert.DoesNotContain(model.tables, t => t.LogicalName == "contoso_alsoelsewhere"); + } + + [Fact] + public void ALookupOutOfTheApp_StillTerminates_SoTheEdgeIsNotLost() + { + var module = new Model.Module { ModuleName = "test" }; + module.entities.AddRange([Entity("contoso_inapp"), Entity("contoso_outside")]); + module.relationships.Add(OneToMany("contoso_inapp", "contoso_inapp_lookup", "contoso_outside")); + + var scope = new ResolvedAppScope { UniqueName = "contoso_app" }; + scope.TableLogicalNames.Add("contoso_inapp"); + + var model = DataModelConverterService.ParseModules([module], scope); + + Assert.Contains(model.relationships, r => r.LeftSideTable?.LogicalName == "contoso_inapp"); + + // NotInApp rather than NotInSolution: this input does declare the table, so it is + // outside the app rather than missing from the solution. + Assert.Contains(model.tables, t => t.LogicalName == "contoso_outside" && t.Type == Model.TableType.NotInApp); + } + + [Fact] + public void WithoutAnAppScope_NothingIsFiltered() + { + var module = new Model.Module { ModuleName = "test" }; + module.entities.AddRange([Entity("contoso_a"), Entity("contoso_b")]); + + var model = DataModelConverterService.ParseModules([module]); + + Assert.Contains(model.tables, t => t.LogicalName == "contoso_a"); + Assert.Contains(model.tables, t => t.LogicalName == "contoso_b"); + } +} diff --git a/tests/TALXIS.CLI.Tests/Data/DataModelConverter/ColumnScopeTests.cs b/tests/TALXIS.CLI.Tests/Data/DataModelConverter/ColumnScopeTests.cs new file mode 100644 index 00000000..9871096c --- /dev/null +++ b/tests/TALXIS.CLI.Tests/Data/DataModelConverter/ColumnScopeTests.cs @@ -0,0 +1,178 @@ +using System; +using System.IO; +using System.Linq; +using System.Xml.Linq; +using TALXIS.CLI.Features.Data.DataModelConverter; +using TALXIS.CLI.Features.Data.DataModelConverter.AppScope; +using Model = TALXIS.CLI.Features.Data.DataModelConverter.Model; +using Xunit; + +namespace TALXIS.CLI.Tests.Data.DataModelConverter; + +/// +/// Narrowing an app's tables to the columns something in it refers to. The rule that must +/// never break: a column an edge depends on stays, because the SQL and EDMX translators +/// read a relationship's endpoints without a null check — dropping one turns a narrower +/// diagram into a crash. +/// +public class ColumnScopeTests : IDisposable +{ + private readonly string _root = Path.Combine(Path.GetTempPath(), "txc-cols-" + Path.GetRandomFileName()); + + public void Dispose() + { + if (Directory.Exists(_root)) Directory.Delete(_root, recursive: true); + GC.SuppressFinalize(this); + } + + private void WriteForm(string contents) + { + var dir = Path.Combine(_root, "module", "Declarations", "Entities", "contoso_thing", "FormXml", "main"); + Directory.CreateDirectory(dir); + File.WriteAllText(Path.Combine(dir, "form.xml"), contents); + } + + private void WriteCode(string fileName, string contents) + { + var dir = Path.Combine(_root, "module", "Plugins"); + Directory.CreateDirectory(dir); + File.WriteAllText(Path.Combine(dir, fileName), contents); + } + + private static XElement Entity(string logicalName, params string[] columns) => + XElement.Parse($""" + + {logicalName} + + primarykey + {string.Join("", columns.Select(c => $"""nvarchar50"""))} + + + """); + + private ResolvedAppScope ScopeFor(params string[] tables) + { + var scope = new ResolvedAppScope { UniqueName = "contoso_app", Detail = DetailLevel.Minimal }; + scope.SearchRoots.Add(_root); + foreach (var t in tables) scope.TableLogicalNames.Add(t); + return scope; + } + + private static Model.Table TableIn(Model.ParsedModel m, string name) => + m.tables.Single(t => t.LogicalName == name); + + [Fact] + public void AColumnAFormRefersTo_IsKept_AndOneNothingRefersTo_IsDroppedAndReported() + { + WriteForm("""
"""); + var module = new Model.Module { ModuleName = "m" }; + module.entities.Add(Entity("contoso_thing", "contoso_onaform", "contoso_nowhere")); + + var scope = ScopeFor("contoso_thing"); + var model = DataModelConverterService.ParseModules([module], scope); + + var columns = TableIn(model, "contoso_thing").Rows.Select(r => r.Name).ToList(); + Assert.Contains("contoso_onaform", columns); + Assert.DoesNotContain("contoso_nowhere", columns); + Assert.Contains(scope.DroppedColumns, c => + c.Table == "contoso_thing" && c.Column == "contoso_nowhere" && c.Reason == DropReason.NoReferenceFound); + } + + [Fact] + public void ThePrimaryKeyIsNeverDropped_EvenWhenNothingRefersToIt() + { + WriteForm("
"); + var module = new Model.Module { ModuleName = "m" }; + module.entities.Add(Entity("contoso_thing", "contoso_nowhere")); + + var model = DataModelConverterService.ParseModules([module], ScopeFor("contoso_thing")); + + Assert.Contains(TableIn(model, "contoso_thing").Rows, r => r.RowType == Model.RowType.Primarykey); + } + + [Fact] + public void AColumnAnEdgeDependsOn_SurvivesAndTheSqlAndEdmxTargetsStillRender() + { + // The regression that matters: the translators dereference a relationship's + // endpoints with no null check, so dropping one crashes rather than shrinks. + WriteForm(""); + var module = new Model.Module { ModuleName = "m" }; + module.entities.Add(Entity("contoso_child", "contoso_lookupid")); + module.entities.Add(Entity("contoso_parent")); + module.relationships.Add(XElement.Parse(""" + + OneToMany + contoso_child + contoso_parent + contoso_lookupid + + """)); + + var model = DataModelConverterService.ParseModules( + [module], ScopeFor("contoso_child", "contoso_parent")); + + Assert.Contains(TableIn(model, "contoso_child").Rows, r => r.Name == "contoso_lookupid"); + Assert.All(model.relationships, r => + { + Assert.NotNull(r.LeftSideRow); + Assert.NotNull(r.RighSideRow); + }); + + Assert.Null(Record.Exception(() => DataModelConverterService.ConvertToSQL(model))); + Assert.Null(Record.Exception(() => DataModelConverterService.ConvertToEDSSQL(model))); + Assert.Null(Record.Exception(() => DataModelConverterService.ConvertToEDMX(model))); + } + + [Fact] + public void AColumnOnlyAPluginMentions_IsKept() + { + // Plug-in and script sources sit outside the declarations, so a column used only + // from one is invisible unless they are read too. + WriteForm(""); + WriteCode("Handler.cs", """var v = entity.GetAttributeValue("contoso_onlyincode");"""); + + var module = new Model.Module { ModuleName = "m" }; + module.entities.Add(Entity("contoso_thing", "contoso_onlyincode")); + + var model = DataModelConverterService.ParseModules([module], ScopeFor("contoso_thing")); + + Assert.Contains(TableIn(model, "contoso_thing").Rows, r => r.Name == "contoso_onlyincode"); + } + + [Fact] + public void AnEntityDeclarationDoesNotCountAsAReferenceToItsOwnColumns() + { + // Scanning Entity.xml would report every column as referenced by its own + // declaration, which would make the filter a no-op that looks like it works. The + // form gives the table an artefact of its own, so the reference rule applies rather + // than the classification a table with no artefacts falls back to. + WriteForm(""""""); + var declarations = Path.Combine(_root, "module", "Declarations", "Entities", "contoso_thing"); + Directory.CreateDirectory(declarations); + File.WriteAllText(Path.Combine(declarations, "Entity.xml"), + Entity("contoso_thing", "contoso_nowhere").ToString()); + + var module = new Model.Module { ModuleName = "m" }; + module.entities.Add(Entity("contoso_thing", "contoso_onaform", "contoso_nowhere")); + + var model = DataModelConverterService.ParseModules([module], ScopeFor("contoso_thing")); + + Assert.DoesNotContain(TableIn(model, "contoso_thing").Rows, r => r.Name == "contoso_nowhere"); + } + + [Fact] + public void AtFullDetail_ColumnsAreLeftAlone() + { + WriteForm("
"); + var module = new Model.Module { ModuleName = "m" }; + module.entities.Add(Entity("contoso_thing", "contoso_nowhere")); + + var scope = new ResolvedAppScope { UniqueName = "contoso_app" }; + scope.TableLogicalNames.Add("contoso_thing"); + + var model = DataModelConverterService.ParseModules([module], scope); + + Assert.Contains(TableIn(model, "contoso_thing").Rows, r => r.Name == "contoso_nowhere"); + Assert.Empty(scope.DroppedColumns); + } +} diff --git a/tests/TALXIS.CLI.Tests/Data/DataModelConverter/DataModelConverterServiceTests.cs b/tests/TALXIS.CLI.Tests/Data/DataModelConverter/DataModelConverterServiceTests.cs new file mode 100644 index 00000000..c764cc66 --- /dev/null +++ b/tests/TALXIS.CLI.Tests/Data/DataModelConverter/DataModelConverterServiceTests.cs @@ -0,0 +1,239 @@ +using System.IO; +using System.Linq; +using System.Reflection; +using DotMake.CommandLine; +using TALXIS.CLI.Features.Data; +using System.Xml.Linq; +using TALXIS.CLI.Features.Data.DataModelConverter; +using Model = TALXIS.CLI.Features.Data.DataModelConverter.Model; +using Xunit; + +namespace TALXIS.CLI.Tests.Data.DataModelConverter; + +/// +/// Regression tests for four defects that made the converter silently lose or corrupt +/// model content. Each was found by diffing converter output against the source +/// declarations of real solutions; each test fails against the unfixed converter. +/// +public class DataModelConverterServiceTests +{ + private static XElement Entity(string logicalName, params string[] attributes) + { + var attrXml = string.Join("", attributes); + return XElement.Parse($""" + + {logicalName} + + + + primarykey + {attrXml} + + + + + """); + } + + private static string Attr(string name, string type) => + $"""{type}"""; + + private static string Lookup(string name) => Attr(name, "lookup"); + + private static XElement OneToMany(string name, string child, string childAttr, string parent) => + XElement.Parse($""" + + OneToMany + {child} + {parent} + {childAttr} + + """); + + private static Model.Module ModuleWith(XElement[] entities, XElement[]? relationships = null) + { + var module = new Model.Module(); + module.entities.AddRange(entities); + if (relationships != null) module.relationships.AddRange(relationships); + return module; + } + + // ---- Defect 1: relationships deduped on the table pair, not the column ------------- + + [Fact] + public void TwoLookupsBetweenSameTablePair_BothProduceRelationships() + { + var module = ModuleWith( + [Entity("account"), Entity("contoso_project", Lookup("contoso_ownerid"), Lookup("contoso_billtoid"))], + [ + OneToMany("rel_owner", "contoso_project", "contoso_ownerid", "account"), + OneToMany("rel_billto", "contoso_project", "contoso_billtoid", "account"), + ]); + + var model = DataModelConverterService.ParseModules([module]); + + var toAccount = model.relationships + .Where(r => r.RighSideTable?.LogicalName == "account") + .Select(r => r.LeftSideRow?.Name) + .ToList(); + + Assert.Equal(2, toAccount.Count); + Assert.Contains("contoso_ownerid", toAccount); + Assert.Contains("contoso_billtoid", toAccount); + } + + // ---- Defect 3: a column vanishes when its option set will not resolve -------------- + + [Fact] + public void PicklistWithUnresolvableOptionSet_KeepsColumnInsteadOfDroppingIt() + { + var picklist = """ + + picklist + contoso_never_declared_anywhere + + """; + var module = ModuleWith([Entity("contoso_thing", picklist)]); + + var table = DataModelConverterService.ParseModules([module]) + .tables.Single(t => t.LogicalName == "contoso_thing"); + + var row = table.Rows.SingleOrDefault(r => r.Name == "contoso_statuscode"); + Assert.NotNull(row); + // Cleared so the column cannot reference an Enum that was never emitted; + // RowType is deliberately left alone so each translator keeps its own handling. + Assert.True(string.IsNullOrEmpty(row!.OptionSetName)); + } + + // ---- Defect 4: output was not reproducible — colours came from new Random() -------- + + [Fact] + public void ModuleColour_IsDerivedFromName_SoConversionIsReproducible() + { + var a = new Model.Module("Areas/Service/Project/Model", new XDocument(new XElement("root"))); + var b = new Model.Module("Areas/Service/Project/Model", new XDocument(new XElement("root"))); + var other = new Model.Module("Areas/Environment/Start/Model", new XDocument(new XElement("root"))); + + Assert.Equal(a.Colorhex, b.Colorhex); + Assert.NotEqual(a.Colorhex, other.Colorhex); + Assert.Matches("^#[0-9A-F]{6}$", a.Colorhex); + } + + // ---- Defect 5: a self-referencing N:N emitted duplicate columns and refs ----------- + + [Fact] + public void SelfReferencingManyToMany_ProducesTwoDistinctIntersectColumns() + { + var manyToMany = XElement.Parse(""" + + ManyToMany + contoso_thing + contoso_thing + contoso_thing_thing + + """); + var module = ModuleWith([Entity("contoso_thing")], [manyToMany]); + + var model = DataModelConverterService.ParseModules([module]); + var intersect = model.tables.Single(t => t.LogicalName == "contoso_thing_thing"); + + var names = intersect.Rows.Select(r => r.Name).ToList(); + Assert.Equal(names.Count, names.Distinct().Count()); + + // Two legs, each anchored on its own column — one shared column produced a + // duplicate endpoint pair, which a DBML parser rejects outright. + var legs = model.relationships.Where(r => r.RighSideTable?.LogicalName == "contoso_thing_thing").ToList(); + Assert.Equal(2, legs.Count); + Assert.Equal(2, legs.Select(l => l.RighSideRow?.Name).Distinct().Count()); + + // The legs also need distinct relationship names: EDMX renders the intersect side + // as NavigationProperty Name="{relationship.Name}", with a matching Partner and + // NavigationPropertyBinding Path, so sharing one name emits each of them twice. + Assert.Equal(2, legs.Select(l => l.Name).Distinct().Count()); + } + + [Fact] + public void SelfReferencingManyToMany_RendersDistinctNavigationPropertiesOnTheIntersect() + { + var manyToMany = XElement.Parse(""" + + ManyToMany + contoso_thing + contoso_thing + contoso_thing_thing + + """); + var module = ModuleWith([Entity("contoso_thing")], [manyToMany]); + + var model = DataModelConverterService.ParseModules([module]); + var edmx = DataModelConverterService.ConvertToEDMX(model); + + // The intersect's own EntityType carries one navigation property per leg. + var intersect = System.Text.RegularExpressions.Regex.Match( + edmx, "", + System.Text.RegularExpressions.RegexOptions.Singleline).Value; + var navNames = System.Text.RegularExpressions.Regex + .Matches(intersect, " m.Groups[1].Value).ToList(); + + Assert.Equal(2, navNames.Count); + Assert.Equal(2, navNames.Distinct().Count()); + } + + // ---- Every relationship endpoint must be non-null: the translators dereference ----- + // ---- LeftSideRow/RighSideRow without a null check. ------------------------------- + + [Fact] + public void EveryEmittedRelationship_HasBothEndpointsResolved() + { + var module = ModuleWith( + [Entity("account"), Entity("contoso_project", Lookup("contoso_ownerid"))], + [OneToMany("rel_owner", "contoso_project", "contoso_ownerid", "account")]); + + var model = DataModelConverterService.ParseModules([module]); + + Assert.All(model.relationships, r => + { + Assert.NotNull(r.LeftSideTable); + Assert.NotNull(r.LeftSideRow); + Assert.NotNull(r.RighSideTable); + Assert.NotNull(r.RighSideRow); + }); + } + + // ---- Every format the CLI advertises must actually convert ------------------------ + // plainsql was listed in the option's AllowedValues and fully implemented in the + // format switch, but missing from the service's SupportedFormats guard -- so it was + // rejected on every invocation, and the error message listed that same guard as truth. + + [Fact] + public void EveryAdvertisedTargetFormat_IsAcceptedByTheService() + { + var advertised = typeof(DataModelConvertCliCommand) + .GetProperty(nameof(DataModelConvertCliCommand.TargetFormat))! + .GetCustomAttribute()! + .AllowedValues! + .Cast() + .ToList(); + + Assert.NotEmpty(advertised); + + var dir = Path.Combine(Path.GetTempPath(), "txc-fmt-" + Path.GetRandomFileName()); + var entityDir = Path.Combine(dir, "Entities", "contoso_thing"); + Directory.CreateDirectory(entityDir); + File.WriteAllText(Path.Combine(entityDir, "Entity.xml"), Entity("contoso_thing").ToString()); + try + { + foreach (var format in advertised) + { + var outFile = Path.Combine(dir, "out." + format); + var ex = Record.Exception(() => DataModelConverterService.ConvertModel(dir, format, outFile)); + Assert.True(ex is null, $"--target {format} is advertised but failed: {ex?.Message}"); + } + } + finally + { + Directory.Delete(dir, recursive: true); + } + } +} diff --git a/tests/TALXIS.CLI.Tests/Data/DataModelConverter/MinimalDetailTableTests.cs b/tests/TALXIS.CLI.Tests/Data/DataModelConverter/MinimalDetailTableTests.cs new file mode 100644 index 00000000..3d40153c --- /dev/null +++ b/tests/TALXIS.CLI.Tests/Data/DataModelConverter/MinimalDetailTableTests.cs @@ -0,0 +1,188 @@ +using System; +using System.IO; +using System.Linq; +using System.Xml.Linq; +using TALXIS.CLI.Features.Data.DataModelConverter; +using TALXIS.CLI.Features.Data.DataModelConverter.AppScope; +using TALXIS.CLI.Features.Data.DataModelConverter.Translators; +using Model = TALXIS.CLI.Features.Data.DataModelConverter.Model; +using Xunit; + +namespace TALXIS.CLI.Tests.Data.DataModelConverter; + +/// +/// The table side of a design view. An N:N belongs to an app's design only when both of its +/// tables do — admitting one on a single side dragged nine systemuser intersects and four +/// far-side stubs into one real app — and a stub for a table the inputs do declare must not +/// be coloured as missing from the solution, which was true of 13 of 14 stubs there. +/// +public class MinimalDetailTableTests : IDisposable +{ + private readonly string _root = Path.Combine(Path.GetTempPath(), "txc-detail-" + Path.GetRandomFileName()); + + public void Dispose() + { + if (Directory.Exists(_root)) Directory.Delete(_root, recursive: true); + GC.SuppressFinalize(this); + } + + private static XElement Entity(string logicalName, params string[] columns) => + XElement.Parse($""" + + {logicalName} + + primarykey + {string.Join("", columns.Select(c => $"""lookup1"""))} + + + """); + + private static XElement ManyToMany(string name, string first, string second) => + XElement.Parse($""" + + ManyToMany + {first} + {second} + {name} + + """); + + private static XElement OneToMany(string name, string referencing, string referenced, string attribute) => + XElement.Parse($""" + + OneToMany + {referencing} + {referenced} + {attribute} + + """); + + private ResolvedAppScope ScopeFor(DetailLevel detail, params string[] tables) + { + var scope = new ResolvedAppScope { UniqueName = "contoso_app", Detail = detail }; + scope.SearchRoots.Add(_root); + foreach (var table in tables) scope.TableLogicalNames.Add(table); + return scope; + } + + private static bool HasTable(Model.ParsedModel model, string name) => + model.tables.Any(t => string.Equals(t.LogicalName, name, StringComparison.OrdinalIgnoreCase)); + + [Fact] + public void AnNToNWithOnlyOneSideInTheApp_IsDroppedInDesign_AndKeptAtFullDetail() + { + // The one gate this change turns. Full detail admits the intersect on either side, + // which is deliberate there; design asks whether the association is the app's own. + Model.ParsedModel Convert(DetailLevel detail) + { + var module = new Model.Module { ModuleName = "m" }; + module.entities.Add(Entity("contoso_inapp")); + module.entities.Add(Entity("contoso_stranger")); + module.relationships.Add(ManyToMany("contoso_inapp_stranger", "contoso_inapp", "contoso_stranger")); + return DataModelConverterService.ParseModules([module], ScopeFor(detail, "contoso_inapp")); + } + + Assert.False(HasTable(Convert(DetailLevel.Minimal), "contoso_inapp_stranger")); + Assert.True(HasTable(Convert(DetailLevel.Full), "contoso_inapp_stranger")); + } + + [Fact] + public void AnNToNWithBothSidesInTheApp_Survives() + { + // The rule must not cost an association the app genuinely owns. + var module = new Model.Module { ModuleName = "m" }; + module.entities.Add(Entity("contoso_left")); + module.entities.Add(Entity("contoso_right")); + module.relationships.Add(ManyToMany("contoso_left_right", "contoso_left", "contoso_right")); + + var model = DataModelConverterService.ParseModules([module], ScopeFor(DetailLevel.Minimal, "contoso_left", "contoso_right")); + + Assert.True(HasTable(model, "contoso_left_right")); + } + + [Fact] + public void DroppingAnNToN_DoesNotRemoveAStubAnOrdinaryLookupStillNeeds() + { + // Suppression is per relationship, not "erase every table a dropped edge touched" -- + // Account and one contract table survived exactly this way in a real app. + var module = new Model.Module { ModuleName = "m" }; + module.entities.Add(Entity("contoso_inapp", "contoso_strangerid")); + module.entities.Add(Entity("contoso_stranger")); + module.relationships.Add(ManyToMany("contoso_inapp_stranger", "contoso_inapp", "contoso_stranger")); + module.relationships.Add(OneToMany("contoso_lookup", "contoso_inapp", "contoso_stranger", "contoso_strangerid")); + + var model = DataModelConverterService.ParseModules([module], ScopeFor(DetailLevel.Minimal, "contoso_inapp")); + + Assert.False(HasTable(model, "contoso_inapp_stranger")); + Assert.True(HasTable(model, "contoso_stranger")); + } + + [Fact] + public void AStubForATableAnInputDeclares_IsMarkedAsOutsideTheApp_AndColouredDifferently() + { + // 13 of 14 stubs in a real app were declared as full entities in the same inputs, so + // the red "not in the solution" was untrue for almost all of them. + var module = new Model.Module { ModuleName = "m" }; + module.entities.Add(Entity("contoso_inapp", "contoso_declaredid")); + module.entities.Add(Entity("contoso_declared")); + module.relationships.Add(OneToMany("contoso_lookup", "contoso_inapp", "contoso_declared", "contoso_declaredid")); + + var model = DataModelConverterService.ParseModules([module], ScopeFor(DetailLevel.Minimal, "contoso_inapp")); + var stub = model.tables.Single(t => t.LogicalName == "contoso_declared"); + + Assert.Equal(Model.TableType.NotInApp, stub.Type); + Assert.Contains("#7f8c8d", stub.ToDbDiagramNotation()); + } + + [Fact] + public void AStubForATableNoInputDeclares_KeepsTheColourThatSaysSo() + { + // The platform's own tables really are absent from the inputs, and a reader needs to + // keep being told that. + var module = new Model.Module { ModuleName = "m" }; + module.entities.Add(Entity("contoso_inapp", "contoso_absentid")); + module.relationships.Add(OneToMany("contoso_lookup", "contoso_inapp", "contoso_absent", "contoso_absentid")); + + var model = DataModelConverterService.ParseModules([module], ScopeFor(DetailLevel.Minimal, "contoso_inapp")); + var stub = model.tables.Single(t => t.LogicalName == "contoso_absent"); + + Assert.Equal(Model.TableType.NotInSolution, stub.Type); + Assert.Contains("#c0392b", stub.ToDbDiagramNotation()); + } + + [Fact] + public void WithoutAnAppScope_NoStubIsEverMarkedAsOutsideOne() + { + // "Outside the app" is only meaningful when an app was named; converting a whole + // solution must keep saying what it says today. + var module = new Model.Module { ModuleName = "m" }; + module.entities.Add(Entity("contoso_thing", "contoso_absentid")); + module.relationships.Add(OneToMany("contoso_lookup", "contoso_thing", "contoso_absent", "contoso_absentid")); + + var model = DataModelConverterService.ParseModules([module], null); + + Assert.DoesNotContain(model.tables, t => t.Type == Model.TableType.NotInApp); + } + + [Fact] + public void EveryTargetStillRendersAfterTablesAndColumnsAreDropped() + { + // The crash class this change risks: the SQL and EDMX translators read a + // relationship's endpoint tables and rows with no null check, so a table may never + // go while an edge still points at it. + var module = new Model.Module { ModuleName = "m" }; + module.entities.Add(Entity("contoso_inapp", "contoso_declaredid")); + module.entities.Add(Entity("contoso_declared")); + module.entities.Add(Entity("contoso_stranger")); + module.relationships.Add(OneToMany("contoso_lookup", "contoso_inapp", "contoso_declared", "contoso_declaredid")); + module.relationships.Add(ManyToMany("contoso_inapp_stranger", "contoso_inapp", "contoso_stranger")); + + var model = DataModelConverterService.ParseModules([module], ScopeFor(DetailLevel.Minimal, "contoso_inapp")); + + Assert.Null(Record.Exception(() => DataModelConverterService.ConvertToDBML(model))); + Assert.Null(Record.Exception(() => DataModelConverterService.ConvertToSQL(model))); + Assert.Null(Record.Exception(() => DataModelConverterService.ConvertToEDSSQL(model))); + Assert.Null(Record.Exception(() => DataModelConverterService.ConvertToEDMX(model))); + Assert.Null(Record.Exception(() => DataModelConverterService.ConvertToRibbonDiff(model))); + } +} diff --git a/tests/TALXIS.CLI.Tests/Data/DataModelConverter/MultipleInputMergeTests.cs b/tests/TALXIS.CLI.Tests/Data/DataModelConverter/MultipleInputMergeTests.cs new file mode 100644 index 00000000..c1fe45ac --- /dev/null +++ b/tests/TALXIS.CLI.Tests/Data/DataModelConverter/MultipleInputMergeTests.cs @@ -0,0 +1,166 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml.Linq; +using TALXIS.CLI.Features.Data.DataModelConverter; +using Model = TALXIS.CLI.Features.Data.DataModelConverter.Model; +using Xunit; + +namespace TALXIS.CLI.Tests.Data.DataModelConverter; + +/// +/// Merging several declaration folders into one model. A delivery project's data model is +/// spread across the modules a product ships plus the project's own layer, and several of +/// them declare part of the same table — so converting each separately and concatenating +/// the output keeps only the first declaration of each table. +/// +public class MultipleInputMergeTests +{ + private static XElement Entity(string logicalName, params string[] attributes) => + XElement.Parse($""" + + {logicalName} + + + + primarykey + {string.Join("", attributes)} + + + + + """); + + private static string Attr(string name, string type, int? maxLength = null) => + $"""{type}{(maxLength is null ? "" : $"{maxLength}")}"""; + + private static Model.Module ModuleOf(string name, params XElement[] entities) + { + var module = new Model.Module { ModuleName = name }; + module.entities.AddRange(entities); + return module; + } + + private static Model.Table TableIn(Model.ParsedModel model, string logicalName) => + model.tables.Single(t => t.LogicalName == logicalName); + + [Fact] + public void TwoModulesDeclaringOneTable_MergeIntoASingleTableWithBothColumns() + { + var a = ModuleOf("base", Entity("contoso_thing", Attr("contoso_fromBase", "nvarchar", 50))); + var b = ModuleOf("layer", Entity("contoso_thing", Attr("contoso_fromLayer", "nvarchar", 50))); + + var model = DataModelConverterService.ParseModules([a, b]); + + Assert.Single(model.tables, t => t.LogicalName == "contoso_thing"); + var names = TableIn(model, "contoso_thing").Rows.Select(r => r.Name).ToList(); + Assert.Contains("contoso_frombase", names.Select(n => n.ToLowerInvariant())); + Assert.Contains("contoso_fromlayer", names.Select(n => n.ToLowerInvariant())); + } + + [Fact] + public void SameAttributeDeclaredInBothModules_ProducesOneColumnNotTwo() + { + var attr = Attr("contoso_shared", "nvarchar", 50); + var model = DataModelConverterService.ParseModules( + [ModuleOf("base", Entity("contoso_thing", attr)), ModuleOf("layer", Entity("contoso_thing", attr))]); + + var rows = TableIn(model, "contoso_thing").Rows + .Where(r => string.Equals(r.Name, "contoso_shared", System.StringComparison.OrdinalIgnoreCase)); + Assert.Single(rows); + } + + [Fact] + public void ConflictingTypeForOneAttribute_KeepsTheFirstInputsDeclaration() + { + var model = DataModelConverterService.ParseModules( + [ + ModuleOf("first", Entity("contoso_thing", Attr("contoso_field", "nvarchar", 50))), + ModuleOf("second", Entity("contoso_thing", Attr("contoso_field", "int"))), + ]); + + var row = TableIn(model, "contoso_thing").Rows + .Single(r => string.Equals(r.Name, "contoso_field", System.StringComparison.OrdinalIgnoreCase)); + Assert.Equal(Model.RowType.Nvarchar, row.RowType); + } + + [Theory] + [InlineData(50, 200, 200)] + [InlineData(200, 50, 200)] + public void DifferingTextLengths_WidenNeverNarrow_RegardlessOfInputOrder(int first, int second, int expected) + { + var model = DataModelConverterService.ParseModules( + [ + ModuleOf("first", Entity("contoso_thing", Attr("contoso_text", "nvarchar", first))), + ModuleOf("second", Entity("contoso_thing", Attr("contoso_text", "nvarchar", second))), + ]); + + var row = TableIn(model, "contoso_thing").Rows + .Single(r => string.Equals(r.Name, "contoso_text", System.StringComparison.OrdinalIgnoreCase)); + Assert.Equal(expected, row.MaxLenght); + } + + [Fact] + public void ModulesAreColouredApart_SoAMergedDiagramShowsWhereEachTableCameFrom() + { + var a = new Model.Module { ModuleName = "src/Modules.Core/Model" }; + var b = new Model.Module { ModuleName = "Areas/Service/Project/Model" }; + + // Assigned through an object initializer, which runs after the constructor body — + // a colour computed in the constructor would be identical for both. + Assert.NotEqual(a.Colorhex, b.Colorhex); + } + + [Fact] + public void OneFolder_ThroughTheListEntryPoint_MatchesTheSingleFolderEntryPoint() + { + var dir = Path.Combine(Path.GetTempPath(), "txc-merge-" + Path.GetRandomFileName()); + var entityDir = Path.Combine(dir, "Entities", "contoso_thing"); + Directory.CreateDirectory(entityDir); + File.WriteAllText(Path.Combine(entityDir, "Entity.xml"), + Entity("contoso_thing", Attr("contoso_field", "nvarchar", 50)).ToString()); + try + { + var single = DataModelConverterService.ParseModelFolder(dir); + var viaList = DataModelConverterService.ParseModelFolders([dir]); + + Assert.Equal(single.tables.Count, viaList.tables.Count); + Assert.Equal(single.relationships.Count, viaList.relationships.Count); + Assert.Equal( + single.tables.Select(t => t.LogicalName).OrderBy(x => x), + viaList.tables.Select(t => t.LogicalName).OrderBy(x => x)); + } + finally + { + Directory.Delete(dir, recursive: true); + } + } + + [Fact] + public void TwoFoldersOnDisk_MergeAttributesAcrossTheFolderBoundary() + { + var root = Path.Combine(Path.GetTempPath(), "txc-merge-" + Path.GetRandomFileName()); + var folders = new List(); + foreach (var (name, attr) in new[] { ("base", "contoso_a"), ("layer", "contoso_b") }) + { + var dir = Path.Combine(root, name, "Declarations"); + Directory.CreateDirectory(Path.Combine(dir, "Entities", "contoso_thing")); + File.WriteAllText(Path.Combine(dir, "Entities", "contoso_thing", "Entity.xml"), + Entity("contoso_thing", Attr(attr, "nvarchar", 50)).ToString()); + folders.Add(dir); + } + try + { + var model = DataModelConverterService.ParseModelFolders(folders); + var names = TableIn(model, "contoso_thing").Rows + .Select(r => r.Name.ToLowerInvariant()).ToList(); + + Assert.Contains("contoso_a", names); + Assert.Contains("contoso_b", names); + } + finally + { + Directory.Delete(root, recursive: true); + } + } +} diff --git a/tests/TALXIS.CLI.Tests/Data/DataModelConverter/PerTableColumnScopeTests.cs b/tests/TALXIS.CLI.Tests/Data/DataModelConverter/PerTableColumnScopeTests.cs new file mode 100644 index 00000000..8d13d2c1 --- /dev/null +++ b/tests/TALXIS.CLI.Tests/Data/DataModelConverter/PerTableColumnScopeTests.cs @@ -0,0 +1,222 @@ +using System; +using System.IO; +using System.Linq; +using System.Xml.Linq; +using TALXIS.CLI.Features.Data.DataModelConverter; +using TALXIS.CLI.Features.Data.DataModelConverter.AppScope; +using Model = TALXIS.CLI.Features.Data.DataModelConverter.Model; +using Xunit; + +namespace TALXIS.CLI.Tests.Data.DataModelConverter; + +/// +/// A reference belongs to the table whose artefact made it. Matching on the name alone keeps +/// a column on every table declaring that name, which is why one view showing createdon kept +/// it on nineteen tables of a real app. +/// +public class PerTableColumnScopeTests : IDisposable +{ + private readonly string _root = Path.Combine(Path.GetTempPath(), "txc-pertable-" + Path.GetRandomFileName()); + + public void Dispose() + { + if (Directory.Exists(_root)) Directory.Delete(_root, recursive: true); + GC.SuppressFinalize(this); + } + + private void WriteArtefact(string entity, string folder, string fileName, string contents) + { + var dir = Path.Combine(_root, "module", "Declarations", "Entities", entity, folder); + Directory.CreateDirectory(dir); + File.WriteAllText(Path.Combine(dir, fileName), contents); + } + + private void WriteSitemap(string contents) + { + var dir = Path.Combine(_root, "module", "Declarations", "AppModuleSiteMaps", "contoso_app"); + Directory.CreateDirectory(dir); + File.WriteAllText(Path.Combine(dir, "AppModuleSiteMap.xml"), contents); + } + + /// An attribute with whatever metadata the case under test needs. + private static string Attribute(string name, string type = "nvarchar", bool? isLogical = null) + { + var flags = string.Empty; + if (isLogical != null) flags += $"{(isLogical.Value ? 1 : 0)}"; + var length = type == "nvarchar" ? "50" : string.Empty; + return $"""{type}{length}{flags}"""; + } + + private static XElement Entity(string logicalName, params string[] attributes) => + XElement.Parse($""" + + {logicalName} + + {Attribute(logicalName + "id", "primarykey")} + {string.Join("", attributes)} + + + """); + + private ResolvedAppScope ScopeFor(DetailLevel detail, string[] tables, params string[] authorPrefixes) + { + var scope = new ResolvedAppScope { UniqueName = "contoso_app", Detail = detail }; + scope.SearchRoots.Add(_root); + foreach (var table in tables) scope.TableLogicalNames.Add(table); + foreach (var prefix in authorPrefixes) scope.AuthorPrefixes.Add(prefix); + return scope; + } + + private static Model.Table TableIn(Model.ParsedModel model, string name) => + model.tables.Single(t => t.LogicalName == name); + + private static bool Has(Model.ParsedModel model, string table, string column) => + TableIn(model, table).Rows.Any(r => string.Equals(r.Name, column, StringComparison.OrdinalIgnoreCase)); + + [Fact] + public void AColumnOnlyOneTablesFormRefersTo_IsKeptThere_AndDroppedOnTheOther() + { + // The whole point of the change. Both tables declare createdon; only one shows it. + WriteArtefact("contoso_shown", "FormXml", "form.xml", + """"""); + WriteArtefact("contoso_hidden", "FormXml", "form.xml", + """
"""); + + var module = new Model.Module { ModuleName = "m" }; + module.entities.Add(Entity("contoso_shown", Attribute("createdon", "datetime"))); + module.entities.Add(Entity("contoso_hidden", Attribute("createdon", "datetime"), + Attribute("contoso_other"))); + + var model = DataModelConverterService.ParseModules([module], ScopeFor(DetailLevel.Minimal, ["contoso_shown", "contoso_hidden"], "contoso")); + + Assert.True(Has(model, "contoso_shown", "createdon")); + Assert.False(Has(model, "contoso_hidden", "createdon")); + } + + [Fact] + public void AnUnattributedReference_CannotRescueAPlatformColumn_ButDoesRescueAnAuthorsOne() + { + // A sitemap belongs to no single table, so it can only be matched by name -- and a + // platform column's name is the same on every table in the org. Letting one rescue + // createdon puts it straight back on every table, which is the defect being fixed. + WriteArtefact("contoso_thing", "FormXml", "form.xml", "
"); + WriteSitemap("""createdon contoso_authored"""); + + var module = new Model.Module { ModuleName = "m" }; + module.entities.Add(Entity("contoso_thing", + Attribute("createdon", "datetime"), + Attribute("contoso_authored"))); + + var model = DataModelConverterService.ParseModules([module], ScopeFor(DetailLevel.Minimal, ["contoso_thing"], "contoso")); + + Assert.False(Has(model, "contoso_thing", "createdon")); + Assert.True(Has(model, "contoso_thing", "contoso_authored")); + } + + [Fact] + public void WithNoPublisherPrefixAvailable_AnUnattributedReferenceStillKeepsAColumn() + { + // No solution manifest, so no publisher prefix to check a name against. Calling a + // column the platform's on that basis would narrow the output on no evidence, so + // the sitemap's reference counts for a name that would otherwise look like the + // platform's. + WriteArtefact("contoso_thing", "FormXml", "form.xml", ""); + WriteSitemap("""mystery_column"""); + + var module = new Model.Module { ModuleName = "m" }; + module.entities.Add(Entity("contoso_thing", Attribute("mystery_column"))); + + var model = DataModelConverterService.ParseModules([module], ScopeFor(DetailLevel.Minimal, ["contoso_thing"])); + + Assert.True(Has(model, "contoso_thing", "mystery_column")); + } + + [Fact] + public void StateAndStatus_SurviveWithNothingReferringToThem() + { + // A state model describes the table whatever shows it, and is the one exception the + // owner named to dropping the platform's own columns. + WriteArtefact("contoso_thing", "FormXml", "form.xml", ""); + + var module = new Model.Module { ModuleName = "m" }; + module.entities.Add(Entity("contoso_thing", + Attribute("statecode", "state"), + Attribute("statuscode", "status"))); + + var model = DataModelConverterService.ParseModules([module], ScopeFor(DetailLevel.Minimal, ["contoso_thing"], "contoso")); + + Assert.True(Has(model, "contoso_thing", "statecode")); + Assert.True(Has(model, "contoso_thing", "statuscode")); + } + + [Fact] + public void DesignPlumbing_IsDroppedEvenWhereAFormRefersToIt() + { + // Logical columns are computed rather than stored, and process-flow bookkeeping is + // not design. Both stay out whatever mentions them, and are reported as such rather + // than as unreferenced. + WriteArtefact("contoso_thing", "FormXml", "form.xml", """ + + + + + """); + + var module = new Model.Module { ModuleName = "m" }; + module.entities.Add(Entity("contoso_thing", + Attribute("owninguser", "lookup", isLogical: true), + Attribute("stageid", "uniqueidentifier"))); + + var scope = ScopeFor(DetailLevel.Minimal, ["contoso_thing"], "contoso"); + var model = DataModelConverterService.ParseModules([module], scope); + + Assert.False(Has(model, "contoso_thing", "owninguser")); + Assert.False(Has(model, "contoso_thing", "stageid")); + Assert.All(scope.DroppedColumns, c => Assert.Equal(DropReason.PlatformPlumbing, c.Reason)); + } + + [Fact] + public void TheBaseCurrencyTwinIsDropped_AndTheColumnItShadowsIsKept() + { + // Both halves are marked as an author's, so nothing but the name pairing separates + // the shadow the platform maintains from the column it shadows. + WriteArtefact("contoso_thing", "FormXml", "form.xml", """ +
+ + + + """); + + var module = new Model.Module { ModuleName = "m" }; + module.entities.Add(Entity("contoso_thing", + Attribute("contoso_cost", "money"), + Attribute("contoso_cost_base", "money"))); + + var model = DataModelConverterService.ParseModules([module], ScopeFor(DetailLevel.Minimal, ["contoso_thing"], "contoso")); + + Assert.True(Has(model, "contoso_thing", "contoso_cost")); + Assert.False(Has(model, "contoso_thing", "contoso_cost_base")); + } + + [Fact] + public void AtFullDetail_EveryDesignOnlyRuleIsInert() + { + // The default must stay exactly what it converts today. + var module = new Model.Module { ModuleName = "m" }; + module.entities.Add(Entity("contoso_thing", + Attribute("createdon", "datetime"), + Attribute("owninguser", "lookup", isLogical: true), + Attribute("stageid", "uniqueidentifier"), + Attribute("contoso_cost", "money"), + Attribute("contoso_cost_base", "money"))); + + var scope = ScopeFor(DetailLevel.Full, ["contoso_thing"], "contoso"); + var model = DataModelConverterService.ParseModules([module], scope); + + Assert.True(Has(model, "contoso_thing", "createdon")); + Assert.True(Has(model, "contoso_thing", "owninguser")); + Assert.True(Has(model, "contoso_thing", "stageid")); + Assert.True(Has(model, "contoso_thing", "contoso_cost_base")); + Assert.Empty(scope.DroppedColumns); + } +}