Skip to content

feat(data): merge several declaration folders into one model #191

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
david-hudec-networg wants to merge 8 commits into
base: master
Choose a base branch
from
Draft

feat(data): merge several declaration folders into one model #191

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
a824822
fix(data): keep every N:1 relationship between the same table pair
david-hudec-networg Sep 1, 2026
544a229
fix(data): keep picklist columns whose option set cannot be resolved
david-hudec-networg Sep 1, 2026
e91f294
fix(data): make conversion reproducible
david-hudec-networg Sep 1, 2026
6d6a6dc
fix(data): give a self-referencing N:N two distinct sides
david-hudec-networg Sep 1, 2026
c040d75
fix(data): accept --target plainsql, which was advertised and always …
david-hudec-networg Sep 1, 2026
e5ab0cd
fix(data): dedup attribute rows when one table is declared more than …
david-hudec-networg Sep 1, 2026
73ac6d1
fix(data): compute the module colour instead of capturing it in the c…
david-hudec-networg Sep 1, 2026
6bdc1e3
feat(data): merge several declaration folders into one model
david-hudec-networg Sep 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions src/TALXIS.CLI.Features.Data/DataModelConvertCliCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,10 @@ public class DataModelConvertCliCommand : TxcLeafCommand
[CliOption(
Name = "--input",
Aliases = ["-i"],
Description = "Path to the input: a solution project folder (.cdsproj/.csproj with SolutionRootPath), a declarations folder, or a .zip solution file. Defaults to the current directory.",
Description = "Path to an input: a solution project folder (.cdsproj/.csproj with SolutionRootPath), a declarations folder, or a .zip solution file. Can be specified multiple times to merge several sources into one model; earlier inputs win where two declare the same attribute differently. Defaults to the current directory.",
Required = false
)]
public string? InputPath { get; set; }
public List<string> InputPaths { get; set; } = [];

[CliOption(
Name = "--target",
Expand All @@ -43,7 +43,7 @@ public class DataModelConvertCliCommand : TxcLeafCommand

protected override Task<int> ExecuteAsync()
{
var inputPath = InputPath ?? Directory.GetCurrentDirectory();
var inputPaths = InputPaths.Count > 0 ? InputPaths : [Directory.GetCurrentDirectory()];
var outputDir = OutputDirectory ?? Path.Combine(Directory.GetCurrentDirectory(), ExportsFolderName);

Directory.CreateDirectory(outputDir);
Expand All @@ -52,7 +52,7 @@ protected override Task<int> ExecuteAsync()
var extension = TargetFormat!.ToLower() == "plainsql" ? "sql" : TargetFormat.ToLower();
var outputFilePath = Path.Combine(outputDir, $"solution.{extension}");

DataModelConverterService.ConvertModel(inputPath, TargetFormat!, outputFilePath);
DataModelConverterService.ConvertModel(inputPaths, TargetFormat!, outputFilePath);

OutputFormatter.WriteResult("succeeded", $"Output written to: {outputFilePath}");
return Task.FromResult(ExitSuccess);
Expand Down
186 changes: 138 additions & 48 deletions src/TALXIS.CLI.Features.Data/DataModelConverter/DataModelConverterService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ namespace TALXIS.CLI.Features.Data.DataModelConverter;
public class DataModelConverterService
{
private static readonly ILogger _logger = TxcLoggerFactory.CreateLogger(nameof(DataModelConverterService));
private static readonly string[] SupportedFormats = ["dbml", "sql", "edmx", "ribbon"];
private static readonly string[] SupportedFormats = ["dbml", "sql", "plainsql", "edmx", "ribbon"];

/// <summary>
/// Parses a Power Platform solution from a solution project folder, a declarations
Expand All @@ -35,29 +35,44 @@ public class DataModelConverterService
/// </list>
/// </remarks>
public static void ConvertModel(string inputPath, string targetFormat, string outputFilePath)
=> ConvertModel([inputPath], targetFormat, outputFilePath);

/// <summary>
/// Converts one or more inputs into a single model. Each input is resolved
/// independently -- a solution project folder, a declarations folder, or a .zip -- and
/// they may be mixed. Earlier inputs take precedence where two declare the same
/// attribute differently.
/// </summary>
public static void ConvertModel(List<string> inputPaths, string targetFormat, string outputFilePath)
{
if (!SupportedFormats.Contains(targetFormat.ToLower()))
throw new ArgumentException($"Unsupported target format '{targetFormat}'. Supported formats are: {string.Join(", ", SupportedFormats)}.");

ParsedModel parsedModel;
if (inputPaths is null || inputPaths.Count == 0)
throw new ArgumentException("At least one input path is required.");

if (Directory.Exists(inputPath))
{
var declarationsPath = ResolveDeclarationsFolder(inputPath);
parsedModel = ParseModelFolder(declarationsPath);
}
else if (File.Exists(inputPath))
{
using var fileStream = new FileStream(inputPath, FileMode.Open, FileAccess.Read);
using var memoryStream = new MemoryStream();
fileStream.CopyTo(memoryStream);
parsedModel = ParseModel(Convert.ToBase64String(memoryStream.ToArray()));
}
else
List<Module> modules = [];
foreach (var inputPath in inputPaths)
{
throw new FileNotFoundException($"Input path '{inputPath}' does not exist.");
if (Directory.Exists(inputPath))
{
modules.Add(ParseFolderIntoModule(ResolveDeclarationsFolder(inputPath)));
}
else if (File.Exists(inputPath))
{
using var fileStream = new FileStream(inputPath, FileMode.Open, FileAccess.Read);
using var memoryStream = new MemoryStream();
fileStream.CopyTo(memoryStream);
modules.Add(ParseZipIntoModule(Convert.ToBase64String(memoryStream.ToArray())));
}
else
{
throw new FileNotFoundException($"Input path '{inputPath}' does not exist.");
}
}

var parsedModel = ParseModules(modules);

var resultString = targetFormat.ToLower() switch
{
"edmx" => ConvertToEDMX(parsedModel),
Expand Down Expand Up @@ -255,11 +270,51 @@ public static string ConvertToEDMX(ParsedModel model)
}

public static ParsedModel ParseModelFolder(string folderPath)
=> ParseModelFolders([folderPath]);

/// <summary>
/// Parses several declarations folders into one model, merging attribute-level.
/// A project's model is rarely one solution: the base product ships several modules
/// that each declare part of a shared table, so converting them separately and
/// concatenating the files loses everything but the first declaration of each table.
/// </summary>
public static ParsedModel ParseModelFolders(List<string> folderPaths)
=> ParseModules([.. folderPaths.Select(ParseFolderIntoModule)]);

/// <summary>
/// Names a module after the folders that own its declarations, so tables can be
/// attributed once several inputs are merged. Several segments are kept because the
/// leaf is almost always "Model" -- one segment would give every input the same name
/// and, with the colour derived from it, the same colour.
/// </summary>
private static string ModuleNameFor(string declarationsFolder)
{
Module module = new();
var full = Path.GetFullPath(declarationsFolder)
.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
var segments = full.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)
.Where(x => x.Length > 0)
.ToList();

// Drop the trailing "Declarations" (or "CDS") folder; it carries no information.
if (segments.Count > 1 && (segments[^1].Equals("Declarations", StringComparison.OrdinalIgnoreCase)
|| segments[^1].Equals("CDS", StringComparison.OrdinalIgnoreCase)))
{
segments.RemoveAt(segments.Count - 1);
}

return string.Join('/', segments.TakeLast(3));
}

private static Module ParseFolderIntoModule(string folderPath)
{
Module module = new() { ModuleName = ModuleNameFor(folderPath) };

// Get files named Entity.xml in subfolders
var entityFiles = Directory.GetFiles(folderPath, "Entity.xml", SearchOption.AllDirectories);
// Ordered: Directory.GetFiles gives no ordering guarantee, so without this the
// table, relationship and enum order in the output varies by filesystem and the
// result cannot be committed or diffed.
var entityFiles = Directory.GetFiles(folderPath, "Entity.xml", SearchOption.AllDirectories)
.OrderBy(f => f, StringComparer.Ordinal).ToArray();

foreach (var file in entityFiles)
{
Expand All @@ -284,7 +339,7 @@ public static ParsedModel ParseModelFolder(string folderPath)
// Get files in folder Other/Relationships (directory may not exist in scaffolded solutions)
var relationshipsDir = Path.Combine(folderPath, "Other", "Relationships");
var relationshipFiles = Directory.Exists(relationshipsDir)
? Directory.GetFiles(relationshipsDir, "*.xml", SearchOption.AllDirectories)
? [.. Directory.GetFiles(relationshipsDir, "*.xml", SearchOption.AllDirectories).OrderBy(f => f, StringComparer.Ordinal)]
: Array.Empty<string>();
foreach (var file in relationshipFiles)
{
Expand All @@ -302,7 +357,7 @@ public static ParsedModel ParseModelFolder(string folderPath)
// Get files in folder called OptionSets (directory may not exist in scaffolded solutions)
var optionsetsDir = Path.Combine(folderPath, "OptionSets");
var optionsetFiles = Directory.Exists(optionsetsDir)
? Directory.GetFiles(optionsetsDir, "*.xml", SearchOption.AllDirectories)
? [.. Directory.GetFiles(optionsetsDir, "*.xml", SearchOption.AllDirectories).OrderBy(f => f, StringComparer.Ordinal)]
: Array.Empty<string>();
foreach (var file in optionsetFiles)
{
Expand All @@ -317,8 +372,24 @@ public static ParsedModel ParseModelFolder(string folderPath)
}
}

return ParseModules([module]);
return module;
}

private static Module ParseZipIntoModule(string base64solution)
{
using ZipArchive archive = new(new MemoryStream(Convert.FromBase64String(base64solution)));

var customizationsxml = archive.Entries.FirstOrDefault(x => x.FullName.Equals("customizations.xml", StringComparison.OrdinalIgnoreCase));
var solutionxml = archive.Entries.FirstOrDefault(x => x.FullName.Equals("solution.xml", StringComparison.OrdinalIgnoreCase));

if (customizationsxml == null || solutionxml == null)
{
throw new FileNotFoundException("The solution archive does not contain the required customizations.xml or solution.xml files.");
}

return new Module(
XDocument.Load(solutionxml.Open()).Descendants().First(x => x.Name == "UniqueName").Value,
XDocument.Load(customizationsxml.Open()));
}

public static ParsedModel ParseModel(string? base64solution)
Expand All @@ -338,19 +409,7 @@ public static ParsedModel ParseModel(List<string> base64solution)

foreach (var solution in base64solution)
{
using ZipArchive archive = new(new MemoryStream(Convert.FromBase64String(solution)));

var customizationsxml = archive.Entries.FirstOrDefault(x => x.FullName.Equals("customizations.xml", StringComparison.OrdinalIgnoreCase));
var solutionxml = archive.Entries.FirstOrDefault(x => x.FullName.Equals("solution.xml", StringComparison.OrdinalIgnoreCase));

if (customizationsxml == null || solutionxml == null)
{
throw new FileNotFoundException("The solution archive does not contain the required customizations.xml or solution.xml files.");
}

Module foundModule = new(XDocument.Load(solutionxml.Open()).Descendants().First(x => x.Name == "UniqueName").Value, XDocument.Load(customizationsxml.Open()));

modules.Add(foundModule);
modules.Add(ParseZipIntoModule(solution));
}

return ParseModules(modules);
Expand All @@ -362,15 +421,20 @@ public static ParsedModel ParseModules(List<Module> modules)
List<Table> EntityTables = ParseEntities(modules);
List<OptionsetEnum> EntityOptionSets = ParseOptionSets(modules);

// Remove optionset rows without optionsets defined
// Downgrade optionset rows whose optionset is not resolvable here (declared with no
// options, owned by another module, or platform-owned) rather than dropping the column.
var validOptionSetNames = EntityOptionSets.Select(x => x.LocalizedName).ToHashSet(StringComparer.OrdinalIgnoreCase);
foreach (var entity in EntityTables)
foreach (var row in EntityTables
.SelectMany(entity => entity.Rows)
.Where(row =>
row.RowType is (RowType.Picklist or RowType.Multiselectoptionset or RowType.State or RowType.Status or RowType.Bit)
&& !validOptionSetNames.Contains(row.OptionSetName)))
{
entity.Rows = [.. entity.Rows
.Where(row =>
row.RowType is not (RowType.Picklist or RowType.Multiselectoptionset or RowType.State or RowType.Status or RowType.Bit)
|| validOptionSetNames.Contains(row.OptionSetName)
)];
// Clearing OptionSetName is enough and is all that is needed: it is what
// ToDbDiagramNotation prefers over RowType, so leaving it set would make the
// column reference an Enum that was never emitted. RowType is deliberately
// left alone so each translator keeps its own handling for the kind.
row.OptionSetName = string.Empty;
}

// Fill in setnames where missing with placeholder logical names
Expand Down Expand Up @@ -420,6 +484,21 @@ public static List<Relationship> ParseRelationships(List<Module> modules, List<T

var intersectEntityName = relationship.Element("IntersectEntityName").Value;

// A self-referencing N:N resolves both sides to the same column name, which
// emitted the column twice and the same Ref twice -- a DBML parser rejects
// both. Dataverse keeps the real per-side names in metadata
// (Entity1/Entity2IntersectAttribute) and they are author-chosen, not
// derivable: the platform's own example pairs connectionroleid with
// associatedconnectionroleid. Solution XML carries neither, and no intersect
// entity declares its own columns, so the second side is suffixed
// positionally rather than guessed.
var firstRowName = firstEntityTable.LogicalName + "id";
var secondRowName = secondEntityTable.LogicalName + "id";
if (string.Equals(firstRowName, secondRowName, StringComparison.OrdinalIgnoreCase))
{
secondRowName = secondEntityTable.LogicalName + "id2";
}

var connectionTable = new Table
{
Type = TableType.ConnectionTable,
Expand All @@ -428,28 +507,39 @@ public static List<Relationship> ParseRelationships(List<Module> modules, List<T
SetName = intersectEntityName + "s",
Rows = {
new TableRow(intersectEntityName + "id", RowType.Primarykey),
new TableRow(firstEntityTable.LogicalName + "id", RowType.Lookup),
new TableRow(secondEntityTable.LogicalName + "id", RowType.Lookup),
new TableRow(firstRowName, RowType.Lookup),
new TableRow(secondRowName, RowType.Lookup),
}
};


EntityTables.Add(connectionTable);

var firstToMid = new Relationship(relationship.Attribute("Name").Value,
// The second leg also needs its own name: both legs otherwise carry the
// relationship name, and EDMX renders the intersect side as
// NavigationProperty Name="{relationship.Name}" plus a matching Partner and
// NavigationPropertyBinding Path, so a self-referencing N:N emits each of
// them twice. Suffixed positionally for the same reason as the column above:
// the real per-side names live in metadata and are author-chosen.
var relationshipName = relationship.Attribute("Name").Value;
var isSelfReferencing = string.Equals(
firstEntityTable.LogicalName, secondEntityTable.LogicalName, StringComparison.OrdinalIgnoreCase);
var secondRelationshipName = isSelfReferencing ? relationshipName + "_2" : relationshipName;

var firstToMid = new Relationship(relationshipName,
"ManyToOne",
firstEntityTable,
firstEntityTable.Rows.FirstOrDefault(x => x.RowType == RowType.Primarykey),
connectionTable,
connectionTable.Rows.FirstOrDefault(x => x.Name == firstEntityTable.LogicalName + "id"));
connectionTable.Rows.FirstOrDefault(x => x.Name == firstRowName));


var secondToMid = new Relationship(relationship.Attribute("Name").Value,
var secondToMid = new Relationship(secondRelationshipName,
"ManyToOne",
secondEntityTable,
secondEntityTable.Rows.FirstOrDefault(x => x.RowType == RowType.Primarykey),
connectionTable,
connectionTable.Rows.FirstOrDefault(x => x.Name == secondEntityTable.LogicalName + "id"));
connectionTable.Rows.FirstOrDefault(x => x.Name == secondRowName));

EntityRelationships.Add(firstToMid);
EntityRelationships.Add(secondToMid);
Expand Down Expand Up @@ -489,7 +579,7 @@ public static List<Relationship> ParseRelationships(List<Module> modules, List<T
rightSideTable,
rightSideTable.Rows.FirstOrDefault(x => x.RowType == RowType.Primarykey));

if (EntityRelationships.FirstOrDefault(x => x.LeftSideTable == entityRelationship.LeftSideTable && x.RighSideTable == entityRelationship.RighSideTable) == default)
if (EntityRelationships.FirstOrDefault(x => x.LeftSideTable == entityRelationship.LeftSideTable && x.LeftSideRow == entityRelationship.LeftSideRow && x.RighSideTable == entityRelationship.RighSideTable) == default)
{
EntityRelationships.Add(entityRelationship);
}
Expand Down
Loading