diff --git a/docs/user/reference/cli/azldev_advanced.md b/docs/user/reference/cli/azldev_advanced.md index 62bac139..b6a5141c 100644 --- a/docs/user/reference/cli/azldev_advanced.md +++ b/docs/user/reference/cli/azldev_advanced.md @@ -37,6 +37,7 @@ output but fully supported. ### SEE ALSO * [azldev](azldev.md) - 🐧 Azure Linux Dev Tool +* [azldev advanced ct-tools](azldev_advanced_ct-tools.md) - Control Tower tools * [azldev advanced mcp](azldev_advanced_mcp.md) - Run in MCP server mode * [azldev advanced mock](azldev_advanced_mock.md) - Run RPM mock tool * [azldev advanced wget](azldev_advanced_wget.md) - Download files via https diff --git a/docs/user/reference/cli/azldev_advanced_ct-tools.md b/docs/user/reference/cli/azldev_advanced_ct-tools.md new file mode 100644 index 00000000..54263745 --- /dev/null +++ b/docs/user/reference/cli/azldev_advanced_ct-tools.md @@ -0,0 +1,40 @@ + + +## azldev advanced ct-tools + +Control Tower tools + +### Synopsis + +Control Tower tools for working with distro configuration. + +Provides utilities for parsing, resolving, and dumping the fully merged +distro configuration used by Control Tower environments. + +### Options + +``` + -h, --help help for ct-tools +``` + +### Options inherited from parent commands + +``` + -y, --accept-all accept all prompts + --color mode output colorization mode {always, auto, never} (default auto) + --config-file stringArray additional TOML config file(s) to merge (may be repeated) + -n, --dry-run dry run only (do not take action) + --network-retries int maximum number of attempts for network operations (minimum 1) (default 3) + --no-default-config disable default configuration + -O, --output-format fmt output format {csv, json, markdown, table} (default table) + --permissive-config do not fail on unknown fields in TOML config files + -C, --project string path to Azure Linux project + -q, --quiet only enable minimal output + -v, --verbose enable verbose output +``` + +### SEE ALSO + +* [azldev advanced](azldev_advanced.md) - Advanced operations +* [azldev advanced ct-tools config-dump](azldev_advanced_ct-tools_config-dump.md) - Dump fully resolved distro config + diff --git a/docs/user/reference/cli/azldev_advanced_ct-tools_config-dump.md b/docs/user/reference/cli/azldev_advanced_ct-tools_config-dump.md new file mode 100644 index 00000000..e1d11b6c --- /dev/null +++ b/docs/user/reference/cli/azldev_advanced_ct-tools_config-dump.md @@ -0,0 +1,54 @@ + + +## azldev advanced ct-tools config-dump + +Dump fully resolved distro config + +### Synopsis + +Parse and resolve all distro configuration TOML files starting from a +top-level file, merge includes, expand all templates (koji-targets, +build-roots, mock-options), and output the fully resolved configuration +filtered to a specific Control Tower environment. + +``` +azldev advanced ct-tools config-dump [flags] +``` + +### Examples + +``` + # Dump config for ct-dev as JSON + azldev advanced ct-tools config-dump \ + --ct-config /path/to/azurelinux.toml \ + --environment ct-dev -O json +``` + +### Options + +``` + --ct-config string Path to the top-level CT distro TOML configuration file + --environment string Control Tower environment name (e.g. ct-dev, ct-staging, ct-prod) + -h, --help help for config-dump +``` + +### Options inherited from parent commands + +``` + -y, --accept-all accept all prompts + --color mode output colorization mode {always, auto, never} (default auto) + --config-file stringArray additional TOML config file(s) to merge (may be repeated) + -n, --dry-run dry run only (do not take action) + --network-retries int maximum number of attempts for network operations (minimum 1) (default 3) + --no-default-config disable default configuration + -O, --output-format fmt output format {csv, json, markdown, table} (default table) + --permissive-config do not fail on unknown fields in TOML config files + -C, --project string path to Azure Linux project + -q, --quiet only enable minimal output + -v, --verbose enable verbose output +``` + +### SEE ALSO + +* [azldev advanced ct-tools](azldev_advanced_ct-tools.md) - Control Tower tools + diff --git a/internal/app/azldev/cmds/advanced/advanced.go b/internal/app/azldev/cmds/advanced/advanced.go index a52741d8..e8c3a8be 100644 --- a/internal/app/azldev/cmds/advanced/advanced.go +++ b/internal/app/azldev/cmds/advanced/advanced.go @@ -23,6 +23,7 @@ output but fully supported.`, } app.AddTopLevelCommand(cmd) + ctToolsOnAppInit(app, cmd) mcpOnAppInit(app, cmd) mockOnAppInit(app, cmd) wgetOnAppInit(app, cmd) diff --git a/internal/app/azldev/cmds/advanced/cttools.go b/internal/app/azldev/cmds/advanced/cttools.go new file mode 100644 index 00000000..e2a4fe17 --- /dev/null +++ b/internal/app/azldev/cmds/advanced/cttools.go @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package advanced + +import ( + "fmt" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/cttools" + "github.com/spf13/cobra" +) + +func ctToolsOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { + parentCmd.AddCommand(NewCTToolsCmd()) +} + +// Constructs a [cobra.Command] for the "ct-tools" subcommand hierarchy. +func NewCTToolsCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "ct-tools", + Short: "Control Tower tools", + Long: `Control Tower tools for working with distro configuration. + +Provides utilities for parsing, resolving, and dumping the fully merged +distro configuration used by Control Tower environments.`, + } + + cmd.AddCommand(NewConfigDumpCmd()) + + return cmd +} + +// Options controlling the config-dump command. +type ConfigDumpOptions struct { + // Path to the top-level TOML configuration file. + ConfigPath string + // The Control Tower environment to filter for (e.g. "ct-dev"). + Environment string +} + +// Constructs a [cobra.Command] for the "ct-tools config-dump" subcommand. +func NewConfigDumpCmd() *cobra.Command { + options := &ConfigDumpOptions{} + + cmd := &cobra.Command{ + Use: "config-dump", + Short: "Dump fully resolved distro config", + Long: `Parse and resolve all distro configuration TOML files starting from a +top-level file, merge includes, expand all templates (koji-targets, +build-roots, mock-options), and output the fully resolved configuration +filtered to a specific Control Tower environment.`, + Example: ` # Dump config for ct-dev as JSON + azldev advanced ct-tools config-dump \ + --ct-config /path/to/azurelinux.toml \ + --environment ct-dev -O json`, + RunE: azldev.RunFuncWithoutRequiredConfig(func(env *azldev.Env) (results interface{}, err error) { + return RunConfigDump(env, options) + }), + } + + cmd.Flags().StringVar( + &options.ConfigPath, "ct-config", "", + "Path to the top-level CT distro TOML configuration file", + ) + + envHelp := "Control Tower environment name " + + "(e.g. ct-dev, ct-staging, ct-prod)" + cmd.Flags().StringVar(&options.Environment, "environment", "", envHelp) + + _ = cmd.MarkFlagRequired("ct-config") + _ = cmd.MarkFlagRequired("environment") + _ = cmd.MarkFlagFilename("ct-config", "toml") + + return cmd +} + +// RunConfigDump loads, resolves, filters, and returns the distro configuration. +func RunConfigDump(env *azldev.Env, options *ConfigDumpOptions) (*cttools.DistroConfig, error) { + config, err := cttools.LoadConfig(env.FS(), options.ConfigPath) + if err != nil { + return nil, fmt.Errorf("failed to load config from %#q:\n%w", options.ConfigPath, err) + } + + if err := cttools.ResolveTemplates(config); err != nil { + return nil, fmt.Errorf("failed to resolve templates:\n%w", err) + } + + if err := cttools.FilterEnvironment(config, options.Environment); err != nil { + return nil, fmt.Errorf("failed to filter environment:\n%w", err) + } + + return config, nil +} diff --git a/internal/app/azldev/cmds/advanced/cttools_test.go b/internal/app/azldev/cmds/advanced/cttools_test.go new file mode 100644 index 00000000..4e7a6855 --- /dev/null +++ b/internal/app/azldev/cmds/advanced/cttools_test.go @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package advanced_test + +import ( + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/cmds/advanced" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewCTToolsCmd(t *testing.T) { + cmd := advanced.NewCTToolsCmd() + require.NotNil(t, cmd) + assert.Equal(t, "ct-tools", cmd.Use) +} + +func TestNewConfigDumpCmd(t *testing.T) { + cmd := advanced.NewConfigDumpCmd() + require.NotNil(t, cmd) + assert.Equal(t, "config-dump", cmd.Use) +} + +func TestCTToolsCmd_HasConfigDumpSubcommand(t *testing.T) { + cmd := advanced.NewCTToolsCmd() + + subCmds := cmd.Commands() + found := false + + for _, sub := range subCmds { + if sub.Use == "config-dump" { + found = true + + break + } + } + + assert.True(t, found, "ct-tools should have config-dump subcommand") +} diff --git a/internal/app/azldev/core/cttools/loader.go b/internal/app/azldev/core/cttools/loader.go new file mode 100644 index 00000000..c31774f9 --- /dev/null +++ b/internal/app/azldev/core/cttools/loader.go @@ -0,0 +1,224 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package cttools + +import ( + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" + + "github.com/bmatcuk/doublestar/v4" + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/pelletier/go-toml/v2" +) + +// LoadConfig loads a distro config starting from the given top-level TOML file path. +// It recursively resolves `include` directives (relative glob paths), deep-merges +// all included files, and returns the final merged [DistroConfig]. +func LoadConfig(fs opctx.FS, topLevelPath string) (*DistroConfig, error) { + absPath, err := filepath.Abs(topLevelPath) + if err != nil { + return nil, fmt.Errorf("failed to resolve absolute path for %#q:\n%w", topLevelPath, err) + } + + slog.Debug("Loading CT distro config", "path", absPath) + + inProgress := make(map[string]bool) + loaded := make(map[string]bool) + + merged, err := loadAndMerge(fs, absPath, inProgress, loaded) + if err != nil { + return nil, err + } + + // Remove the include key from the merged map before marshalling to typed struct. + delete(merged, "include") + + // Re-serialize the merged map to TOML, then unmarshal into the typed struct. + buf, err := toml.Marshal(merged) + if err != nil { + return nil, fmt.Errorf("failed to marshal merged config:\n%w", err) + } + + var config DistroConfig + if err := toml.Unmarshal(buf, &config); err != nil { + return nil, fmt.Errorf("failed to unmarshal merged config into typed struct:\n%w", err) + } + + return &config, nil +} + +// loadAndMerge loads a single TOML file, processes its include directives, +// and returns the deep-merged result as a raw map. inProgress tracks the current +// recursion stack to detect cycles. loaded tracks files that have already been +// fully processed so that diamond includes (e.g. common.toml included from two +// branches) are only merged once. +func loadAndMerge( + fs opctx.FS, absPath string, inProgress map[string]bool, loaded map[string]bool, +) (map[string]any, error) { + if inProgress[absPath] { + return nil, fmt.Errorf("circular include detected for %#q", absPath) + } + + // Skip files already loaded from another branch to avoid duplicate merging. + if loaded[absPath] { + slog.Debug("Skipping already-loaded CT config file", "path", absPath) + + return make(map[string]any), nil + } + + inProgress[absPath] = true + defer delete(inProgress, absPath) + + slog.Debug("Loading CT config file", "path", absPath) + + data, err := fileutils.ReadFile(fs, absPath) + if err != nil { + return nil, fmt.Errorf("failed to read %#q:\n%w", absPath, err) + } + + var raw map[string]any + if err := toml.Unmarshal(data, &raw); err != nil { + return nil, fmt.Errorf("failed to parse TOML %#q:\n%w", absPath, err) + } + + // Extract and process includes. + includes, err := extractIncludes(raw, absPath) + if err != nil { + return nil, err + } + + // Start with the current file's data (without include key). + result := make(map[string]any) + deepMergeMaps(result, raw) + delete(result, "include") + + if err := resolveIncludes(fs, absPath, includes, inProgress, loaded, result); err != nil { + return nil, err + } + + loaded[absPath] = true + + return result, nil +} + +// resolveIncludes processes include directives for a single config file, loading +// and deep-merging each included file's content into result. +func resolveIncludes( + fs opctx.FS, absPath string, includes []string, + inProgress map[string]bool, loaded map[string]bool, result map[string]any, +) error { + dir := filepath.Dir(absPath) + + for _, pattern := range includes { + globPath := filepath.Join(dir, pattern) + + slog.Debug("Resolving CT config include", "pattern", globPath, "from", absPath) + + matches, err := fileutils.Glob(fs, globPath, doublestar.WithFilesOnly()) + if err != nil { + return fmt.Errorf("failed to glob %#q (from include in %#q):\n%w", globPath, absPath, err) + } + + if len(matches) == 0 && !containsGlobMeta(pattern) { + return fmt.Errorf( + "failed to find include file %#q referenced in %#q:\n%w", + pattern, absPath, os.ErrNotExist, + ) + } + + for _, match := range matches { + // fileutils.Glob may return paths relative to the FS root. + // Ensure they are absolute for consistent handling. + matchAbs := match + if !filepath.IsAbs(match) { + matchAbs = "/" + match + } + + // Skip self-includes (e.g., when a glob like "./*.toml" matches the current file). + if matchAbs == absPath { + continue + } + + child, err := loadAndMerge(fs, matchAbs, inProgress, loaded) + if err != nil { + return fmt.Errorf("error loading include %#q from %#q:\n%w", matchAbs, absPath, err) + } + + deepMergeMaps(result, child) + } + } + + return nil +} + +// extractIncludes reads the "include" key from a raw TOML map and returns it as a string slice. +func extractIncludes(raw map[string]any, filePath string) ([]string, error) { + includeVal, hasInclude := raw["include"] + if !hasInclude { + return nil, nil + } + + includeSlice, isSlice := includeVal.([]any) + if !isSlice { + return nil, fmt.Errorf("'include' in %#q must be an array of strings", filePath) + } + + result := make([]string, 0, len(includeSlice)) + + for _, v := range includeSlice { + s, ok := v.(string) + if !ok { + return nil, fmt.Errorf("'include' entry in %#q must be a string, got %T", filePath, v) + } + + result = append(result, s) + } + + return result, nil +} + +// containsGlobMeta reports whether the pattern contains glob metacharacters. +func containsGlobMeta(pattern string) bool { + return strings.ContainsAny(pattern, "*?[") +} + +// deepMergeMaps merges src into dst recursively. For map values, sub-maps are merged recursively. +// For slice values, slices are concatenated. For all other types, src overwrites dst. +func deepMergeMaps(dst, src map[string]any) { + for key, srcVal := range src { + dstVal, exists := dst[key] + if !exists { + dst[key] = srcVal + + continue + } + + // If both are maps, merge recursively. + srcMap, srcIsMap := srcVal.(map[string]any) + dstMap, dstIsMap := dstVal.(map[string]any) + + if srcIsMap && dstIsMap { + deepMergeMaps(dstMap, srcMap) + + continue + } + + // If both are slices, concatenate. + srcSlice, srcIsSlice := srcVal.([]any) + dstSlice, dstIsSlice := dstVal.([]any) + + if srcIsSlice && dstIsSlice { + dst[key] = append(dstSlice, srcSlice...) + + continue + } + + // Otherwise, src overwrites dst. + dst[key] = srcVal + } +} diff --git a/internal/app/azldev/core/cttools/loader_test.go b/internal/app/azldev/core/cttools/loader_test.go new file mode 100644 index 00000000..38ec935b --- /dev/null +++ b/internal/app/azldev/core/cttools/loader_test.go @@ -0,0 +1,286 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package cttools_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/cttools" + "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testConfigDir = "/testconfig" + +func TestLoadConfig_SimpleFile(t *testing.T) { + ctx := testctx.NewCtx() + mainPath := filepath.Join(testConfigDir, "main.toml") + + require.NoError(t, fileutils.MkdirAll(ctx.FS(), testConfigDir)) + require.NoError(t, fileutils.WriteFile(ctx.FS(), mainPath, []byte(` +[distros.testdistro] +description = "Test Distro" +`), fileperms.PrivateFile)) + + config, err := cttools.LoadConfig(ctx.FS(), mainPath) + require.NoError(t, err) + require.Contains(t, config.Distros, "testdistro") + assert.Equal(t, "Test Distro", config.Distros["testdistro"].Description) +} + +func TestLoadConfig_IncludeResolution(t *testing.T) { + ctx := testctx.NewCtx() + + require.NoError(t, fileutils.MkdirAll(ctx.FS(), testConfigDir)) + + mainPath := filepath.Join(testConfigDir, "main.toml") + require.NoError(t, fileutils.WriteFile(ctx.FS(), mainPath, []byte(` +include = ["sub.toml"] + +[distros.testdistro] +description = "Test Distro" +`), fileperms.PrivateFile)) + + subPath := filepath.Join(testConfigDir, "sub.toml") + require.NoError(t, fileutils.WriteFile(ctx.FS(), subPath, []byte(` +[mock-options-templates.rpm] +options = ["opt1", "opt2"] +`), fileperms.PrivateFile)) + + config, err := cttools.LoadConfig(ctx.FS(), mainPath) + require.NoError(t, err) + + require.Contains(t, config.Distros, "testdistro") + require.Contains(t, config.MockOptionsTemplates, "rpm") + assert.Equal(t, []string{"opt1", "opt2"}, config.MockOptionsTemplates["rpm"].Options) +} + +func TestLoadConfig_NestedIncludes(t *testing.T) { + ctx := testctx.NewCtx() + subDir := filepath.Join(testConfigDir, "sub") + + require.NoError(t, fileutils.MkdirAll(ctx.FS(), subDir)) + + mainPath := filepath.Join(testConfigDir, "main.toml") + require.NoError(t, fileutils.WriteFile(ctx.FS(), mainPath, []byte(` +include = ["sub/mid.toml"] + +[distros.d] +description = "D" +`), fileperms.PrivateFile)) + + midPath := filepath.Join(subDir, "mid.toml") + require.NoError(t, fileutils.WriteFile(ctx.FS(), midPath, []byte(` +include = ["leaf.toml"] + +[mock-options-templates.rpm] +options = ["a"] +`), fileperms.PrivateFile)) + + leafPath := filepath.Join(subDir, "leaf.toml") + require.NoError(t, fileutils.WriteFile(ctx.FS(), leafPath, []byte(` +[build-root-templates.srpm] +packages = ["bash"] +`), fileperms.PrivateFile)) + + config, err := cttools.LoadConfig(ctx.FS(), mainPath) + require.NoError(t, err) + + require.Contains(t, config.Distros, "d") + require.Contains(t, config.MockOptionsTemplates, "rpm") + require.Contains(t, config.BuildRootTemplates, "srpm") + assert.Equal(t, []string{"bash"}, config.BuildRootTemplates["srpm"].Packages) +} + +func TestLoadConfig_GlobIncludes(t *testing.T) { + ctx := testctx.NewCtx() + tmplDir := filepath.Join(testConfigDir, "templates") + + require.NoError(t, fileutils.MkdirAll(ctx.FS(), tmplDir)) + + mainPath := filepath.Join(testConfigDir, "main.toml") + require.NoError(t, fileutils.WriteFile(ctx.FS(), mainPath, []byte(` +include = ["templates/*.toml"] +`), fileperms.PrivateFile)) + + require.NoError(t, fileutils.WriteFile(ctx.FS(), filepath.Join(tmplDir, "mock.toml"), []byte(` +[mock-options-templates.rpm] +options = ["opt1"] +`), fileperms.PrivateFile)) + + require.NoError(t, fileutils.WriteFile(ctx.FS(), filepath.Join(tmplDir, "build.toml"), []byte(` +[build-root-templates.srpm] +packages = ["bash"] +`), fileperms.PrivateFile)) + + config, err := cttools.LoadConfig(ctx.FS(), mainPath) + require.NoError(t, err) + + require.Contains(t, config.MockOptionsTemplates, "rpm") + require.Contains(t, config.BuildRootTemplates, "srpm") +} + +func TestLoadConfig_DeepMerge_MapsMerge(t *testing.T) { + ctx := testctx.NewCtx() + + require.NoError(t, fileutils.MkdirAll(ctx.FS(), testConfigDir)) + + mainPath := filepath.Join(testConfigDir, "main.toml") + require.NoError(t, fileutils.WriteFile(ctx.FS(), mainPath, []byte(` +include = ["extra.toml"] + +[distros.d1] +description = "D1" +`), fileperms.PrivateFile)) + + extraPath := filepath.Join(testConfigDir, "extra.toml") + require.NoError(t, fileutils.WriteFile(ctx.FS(), extraPath, []byte(` +[distros.d2] +description = "D2" +`), fileperms.PrivateFile)) + + config, err := cttools.LoadConfig(ctx.FS(), mainPath) + require.NoError(t, err) + + require.Contains(t, config.Distros, "d1") + require.Contains(t, config.Distros, "d2") +} + +func TestLoadConfig_DeepMerge_ArraysConcatenate(t *testing.T) { + ctx := testctx.NewCtx() + + require.NoError(t, fileutils.MkdirAll(ctx.FS(), testConfigDir)) + + mainPath := filepath.Join(testConfigDir, "main.toml") + require.NoError(t, fileutils.WriteFile(ctx.FS(), mainPath, []byte(` +include = ["extra.toml"] + +[[distros.d.shadow-allowlists]] +tag-name = "tag1" +`), fileperms.PrivateFile)) + + extraPath := filepath.Join(testConfigDir, "extra.toml") + require.NoError(t, fileutils.WriteFile(ctx.FS(), extraPath, []byte(` +[[distros.d.shadow-allowlists]] +tag-name = "tag2" +`), fileperms.PrivateFile)) + + config, err := cttools.LoadConfig(ctx.FS(), mainPath) + require.NoError(t, err) + + require.Contains(t, config.Distros, "d") + + allowlists := config.Distros["d"].ShadowAllowlists + require.Len(t, allowlists, 2) + assert.Equal(t, "tag1", allowlists[0].TagName) + assert.Equal(t, "tag2", allowlists[1].TagName) +} + +func TestLoadConfig_CircularInclude(t *testing.T) { + ctx := testctx.NewCtx() + + require.NoError(t, fileutils.MkdirAll(ctx.FS(), testConfigDir)) + + aPath := filepath.Join(testConfigDir, "a.toml") + bPath := filepath.Join(testConfigDir, "b.toml") + + require.NoError(t, fileutils.WriteFile(ctx.FS(), aPath, []byte(`include = ["b.toml"]`), fileperms.PrivateFile)) + require.NoError(t, fileutils.WriteFile(ctx.FS(), bPath, []byte(`include = ["a.toml"]`), fileperms.PrivateFile)) + + _, err := cttools.LoadConfig(ctx.FS(), aPath) + require.Error(t, err) + assert.Contains(t, err.Error(), "circular include") +} + +func TestLoadConfig_DiamondInclude(t *testing.T) { + // A diamond include pattern: root includes both a.toml and b.toml, + // and both a.toml and b.toml include common.toml. This must not be + // treated as a circular include. + ctx := testctx.NewCtx() + + require.NoError(t, fileutils.MkdirAll(ctx.FS(), testConfigDir)) + + rootPath := filepath.Join(testConfigDir, "root.toml") + aPath := filepath.Join(testConfigDir, "a.toml") + bPath := filepath.Join(testConfigDir, "b.toml") + commonPath := filepath.Join(testConfigDir, "common.toml") + + require.NoError(t, fileutils.WriteFile(ctx.FS(), rootPath, + []byte(`include = ["a.toml", "b.toml"]`), fileperms.PrivateFile)) + require.NoError(t, fileutils.WriteFile(ctx.FS(), aPath, + []byte(`include = ["common.toml"]`), fileperms.PrivateFile)) + require.NoError(t, fileutils.WriteFile(ctx.FS(), bPath, + []byte(`include = ["common.toml"]`), fileperms.PrivateFile)) + require.NoError(t, fileutils.WriteFile(ctx.FS(), commonPath, []byte(` +[mock-options-templates.shared] +options = ["shared-opt"] +`), fileperms.PrivateFile)) + + config, err := cttools.LoadConfig(ctx.FS(), rootPath) + require.NoError(t, err) + require.Contains(t, config.MockOptionsTemplates, "shared") + assert.Equal(t, []string{"shared-opt"}, config.MockOptionsTemplates["shared"].Options) +} + +func TestLoadConfig_MissingInclude(t *testing.T) { + ctx := testctx.NewCtx() + + require.NoError(t, fileutils.MkdirAll(ctx.FS(), testConfigDir)) + + mainPath := filepath.Join(testConfigDir, "main.toml") + content := []byte(`include = ["nonexistent.toml"]`) + require.NoError(t, fileutils.WriteFile(ctx.FS(), mainPath, content, fileperms.PrivateFile)) + + // Non-glob include that doesn't exist should produce an error. + _, err := cttools.LoadConfig(ctx.FS(), mainPath) + require.Error(t, err) + assert.ErrorIs(t, err, os.ErrNotExist) +} + +func TestLoadConfig_MissingGlobInclude(t *testing.T) { + ctx := testctx.NewCtx() + + require.NoError(t, fileutils.MkdirAll(ctx.FS(), testConfigDir)) + + mainPath := filepath.Join(testConfigDir, "main.toml") + content := []byte(`include = ["nonexistent/*.toml"]`) + require.NoError(t, fileutils.WriteFile(ctx.FS(), mainPath, content, fileperms.PrivateFile)) + + // Glob pattern with no matches should silently succeed. + config, err := cttools.LoadConfig(ctx.FS(), mainPath) + require.NoError(t, err) + assert.Empty(t, config.Distros) +} + +func TestLoadConfig_InvalidTOML(t *testing.T) { + ctx := testctx.NewCtx() + + require.NoError(t, fileutils.MkdirAll(ctx.FS(), testConfigDir)) + + mainPath := filepath.Join(testConfigDir, "main.toml") + content := []byte(`this is not valid toml {{{`) + require.NoError(t, fileutils.WriteFile(ctx.FS(), mainPath, content, fileperms.PrivateFile)) + + _, err := cttools.LoadConfig(ctx.FS(), mainPath) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to parse TOML") +} + +func TestLoadConfig_InvalidIncludeType(t *testing.T) { + ctx := testctx.NewCtx() + + require.NoError(t, fileutils.MkdirAll(ctx.FS(), testConfigDir)) + + mainPath := filepath.Join(testConfigDir, "main.toml") + require.NoError(t, fileutils.WriteFile(ctx.FS(), mainPath, []byte(`include = 42`), fileperms.PrivateFile)) + + _, err := cttools.LoadConfig(ctx.FS(), mainPath) + require.Error(t, err) + assert.Contains(t, err.Error(), "must be an array") +} diff --git a/internal/app/azldev/core/cttools/resolver.go b/internal/app/azldev/core/cttools/resolver.go new file mode 100644 index 00000000..a3393a8c --- /dev/null +++ b/internal/app/azldev/core/cttools/resolver.go @@ -0,0 +1,211 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package cttools + +import ( + "fmt" + "sort" +) + +// ResolveTemplates resolves all koji target templates for every git-source-repo in every distro +// version. It expands build-root references, mock-options references, and applies +// environment-prefix / repo-prefix / parent-prefix to produce [ResolvedKojiTarget] entries. +func ResolveTemplates(config *DistroConfig) error { + for _, distroName := range sortedKeys(config.Distros) { + distro := config.Distros[distroName] + + for _, versionName := range sortedKeys(distro.Versions) { + version := distro.Versions[versionName] + + for _, repoName := range sortedKeys(version.GitSourceRepos) { + repos := version.GitSourceRepos[repoName] + + for i := range repos { + repo := &repos[i] + + if err := resolveRepoTargets(config, &version, repo); err != nil { + return fmt.Errorf("error resolving targets for distro %#q, version %#q, repo %#q:\n%w", + distroName, versionName, repoName, err) + } + } + + version.GitSourceRepos[repoName] = repos + } + + distro.Versions[versionName] = version + } + + config.Distros[distroName] = distro + } + + return nil +} + +// FilterEnvironment removes all environments from the config except the specified one. +func FilterEnvironment(config *DistroConfig, envName string) error { + env, ok := config.Environments[envName] + if !ok { + available := sortedKeys(config.Environments) + + return fmt.Errorf("environment %#q not found; available: %v", envName, available) + } + + config.Environments = map[string]Environment{ + envName: env, + } + + return nil +} + +// resolveRepoTargets resolves all koji targets for a single git-source-repo. +func resolveRepoTargets(config *DistroConfig, version *Version, repo *GitSourceRepo) error { + templateSetName := repo.KojiTargets + + templateSet, ok := config.KojiTargetsTemplates[templateSetName] + if !ok { + return fmt.Errorf("koji-targets-template %#q not found", templateSetName) + } + + envPrefix := version.EnvironmentPrefix + repoPrefix := repo.RepoPrefix + parentPrefix := repo.ParentPrefix + + var resolved []ResolvedKojiTarget + + for _, targetName := range sortedKeys(templateSet) { + targets := templateSet[targetName] + + for _, tmpl := range targets { + resolvedTarget, err := resolveOneTarget( + config, version, envPrefix, repoPrefix, parentPrefix, targetName, &tmpl, + ) + if err != nil { + return fmt.Errorf("error resolving target %#q:\n%w", targetName, err) + } + + resolved = append(resolved, *resolvedTarget) + } + } + + sort.Slice(resolved, func(i, j int) bool { + return resolved[i].Name < resolved[j].Name + }) + + repo.ResolvedKojiTargets = resolved + + return nil +} + +// resolveOneTarget resolves a single koji target template into a [ResolvedKojiTarget]. +func resolveOneTarget( + config *DistroConfig, + version *Version, + envPrefix, repoPrefix, parentPrefix, targetName string, + tmpl *KojiTarget, +) (*ResolvedKojiTarget, error) { + resolvedName := applyPrefix(envPrefix, repoPrefix, targetName) + resolvedOutputTag := applyPrefix(envPrefix, repoPrefix, tmpl.OutputTag) + + var resolvedParentTag string + if tmpl.ParentTag != "" { + resolvedParentTag = applyPrefix(parentPrefix, "", tmpl.ParentTag) + } + + // Resolve build roots. + buildRoots, err := resolveBuildRoots(config, tmpl.BuildRoots) + if err != nil { + return nil, err + } + + // Resolve mock options. + mockOpts, err := resolveMockOptions(config, tmpl.MockOptionsBase) + if err != nil { + return nil, err + } + + // Resolve mock-dist-tag (dereference field name on the version). + var mockDistTag string + if tmpl.MockDistTag != "" { + mockDistTag, err = resolveDistTag(version, tmpl.MockDistTag) + if err != nil { + return nil, err + } + } + + resolvedTarget := &ResolvedKojiTarget{ + Name: resolvedName, + OutputTag: resolvedOutputTag, + ParentTag: resolvedParentTag, + BuildRoots: buildRoots, + MockOptions: mockOpts, + MockDistTag: mockDistTag, + ExternalRepos: tmpl.ExternalRepos, + } + + return resolvedTarget, nil +} + +// applyPrefix constructs "{envPrefix}-{repoPrefix}-{suffix}" or "{envPrefix}-{suffix}" when +// repoPrefix is empty. +func applyPrefix(envPrefix, repoPrefix, suffix string) string { + if repoPrefix != "" { + return envPrefix + "-" + repoPrefix + "-" + suffix + } + + return envPrefix + "-" + suffix +} + +// resolveBuildRoots expands build-root references to their package lists. +func resolveBuildRoots(config *DistroConfig, refs []BuildRootRef) ([]ResolvedBuildRoot, error) { + resolved := make([]ResolvedBuildRoot, 0, len(refs)) + + for _, ref := range refs { + tmpl, ok := config.BuildRootTemplates[ref.Value] + if !ok { + return nil, fmt.Errorf("build-root-template %#q not found", ref.Value) + } + + resolved = append(resolved, ResolvedBuildRoot{ + Type: ref.Type, + Packages: tmpl.Packages, + }) + } + + return resolved, nil +} + +// resolveMockOptions looks up a mock-options template by name and returns its options. +func resolveMockOptions(config *DistroConfig, templateName string) ([]string, error) { + tmpl, ok := config.MockOptionsTemplates[templateName] + if !ok { + return nil, fmt.Errorf("mock-options-template %#q not found", templateName) + } + + return tmpl.Options, nil +} + +// resolveDistTag maps a mock-dist-tag field name (e.g., "rpm-macro-dist") to the corresponding +// value on the [Version]. +func resolveDistTag(version *Version, fieldName string) (string, error) { + switch fieldName { + case "rpm-macro-dist": + return version.RPMMacroDist, nil + case "rpm-macro-dist-bootstrap": + return version.RPMMacroDistBootstrap, nil + default: + return "", fmt.Errorf("unknown mock-dist-tag field %#q", fieldName) + } +} + +// sortedKeys returns the keys of a string-keyed map in sorted order. +func sortedKeys[V any](inputMap map[string]V) []string { + keys := make([]string, 0, len(inputMap)) + for key := range inputMap { + keys = append(keys, key) + } + + sort.Strings(keys) + + return keys +} diff --git a/internal/app/azldev/core/cttools/resolver_test.go b/internal/app/azldev/core/cttools/resolver_test.go new file mode 100644 index 00000000..a56aceb6 --- /dev/null +++ b/internal/app/azldev/core/cttools/resolver_test.go @@ -0,0 +1,275 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package cttools_test + +import ( + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/cttools" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResolveTemplates_BasicResolution(t *testing.T) { + config := &cttools.DistroConfig{ + Distros: map[string]cttools.Distro{ + "testdistro": { + Description: "Test", + Versions: map[string]cttools.Version{ + "1.0": { + Description: "v1.0", + ReleaseVer: "1.0", + EnvironmentPrefix: "td1-dev", + RPMMacroDist: ".td1-dev", + GitSourceRepos: map[string][]cttools.GitSourceRepo{ + "main": { + { + Ref: "https://example.com/repo.git", + DefaultBranch: "main", + DefaultKojiRPMTarget: "td1-dev-rpms-target", + KojiTargets: "base", + ParentPrefix: "td1-dev", + }, + }, + }, + }, + }, + }, + }, + KojiTargetsTemplates: map[string]map[string][]cttools.KojiTarget{ + "base": { + "rpms-target": { + { + OutputTag: "rpms-tag", + ParentTag: "bootstrap-rpms-tag", + BuildRoots: []cttools.BuildRootRef{{Type: "build", Value: "rpm"}}, + MockOptionsBase: "rpm", + MockDistTag: "rpm-macro-dist", + }, + }, + }, + }, + MockOptionsTemplates: map[string]cttools.MockOptionsTemplate{ + "rpm": {Options: []string{"opt1", "opt2"}}, + }, + BuildRootTemplates: map[string]cttools.BuildRootTemplate{ + "rpm": {Packages: []string{"bash", "gcc"}}, + }, + } + + err := cttools.ResolveTemplates(config) + require.NoError(t, err) + + repos := config.Distros["testdistro"].Versions["1.0"].GitSourceRepos["main"] + require.Len(t, repos, 1) + + resolved := repos[0].ResolvedKojiTargets + require.Len(t, resolved, 1) + + target := resolved[0] + assert.Equal(t, "td1-dev-rpms-target", target.Name) + assert.Equal(t, "td1-dev-rpms-tag", target.OutputTag) + assert.Equal(t, "td1-dev-bootstrap-rpms-tag", target.ParentTag) + assert.Equal(t, []string{"opt1", "opt2"}, target.MockOptions) + assert.Equal(t, ".td1-dev", target.MockDistTag) + + require.Len(t, target.BuildRoots, 1) + assert.Equal(t, "build", target.BuildRoots[0].Type) + assert.Equal(t, []string{"bash", "gcc"}, target.BuildRoots[0].Packages) +} + +func TestResolveTemplates_WithRepoPrefix(t *testing.T) { + config := &cttools.DistroConfig{ + Distros: map[string]cttools.Distro{ + "d": { + Description: "D", + Versions: map[string]cttools.Version{ + "1.0": { + Description: "v1.0", + ReleaseVer: "1.0", + EnvironmentPrefix: "azl4-dev", + RPMMacroDist: ".azl4-dev", + RPMMacroDistBootstrap: ".azl4-dev~bootstrap", + GitSourceRepos: map[string][]cttools.GitSourceRepo{ + "nvidia": { + { + Ref: "https://example.com/nvidia.git", + DefaultBranch: "main", + DefaultKojiRPMTarget: "azl4-dev-nvidia-rpms-target", + KojiTargets: "prop", + RepoPrefix: "nvidia", + ParentPrefix: "azl4-dev", + }, + }, + }, + }, + }, + }, + }, + KojiTargetsTemplates: map[string]map[string][]cttools.KojiTarget{ + "prop": { + "bootstrap-rpms-target": { + { + OutputTag: "bootstrap-rpms-tag", + ParentTag: "bootstrap-rpms-tag", + BuildRoots: []cttools.BuildRootRef{{Type: "build", Value: "rpm"}}, + MockOptionsBase: "rpm", + MockDistTag: "rpm-macro-dist-bootstrap", + }, + }, + "rpms-target": { + { + OutputTag: "rpms-tag", + ParentTag: "rpms-tag", + BuildRoots: []cttools.BuildRootRef{{Type: "build", Value: "rpm"}}, + MockOptionsBase: "rpm", + MockDistTag: "rpm-macro-dist", + }, + }, + }, + }, + MockOptionsTemplates: map[string]cttools.MockOptionsTemplate{ + "rpm": {Options: []string{"opt1"}}, + }, + BuildRootTemplates: map[string]cttools.BuildRootTemplate{ + "rpm": {Packages: []string{"bash"}}, + }, + } + + err := cttools.ResolveTemplates(config) + require.NoError(t, err) + + repos := config.Distros["d"].Versions["1.0"].GitSourceRepos["nvidia"] + require.Len(t, repos, 1) + + resolved := repos[0].ResolvedKojiTargets + require.Len(t, resolved, 2) + + names := make(map[string]cttools.ResolvedKojiTarget, len(resolved)) + for _, rt := range resolved { + names[rt.Name] = rt + } + + bootstrap := names["azl4-dev-nvidia-bootstrap-rpms-target"] + assert.Equal(t, "azl4-dev-nvidia-bootstrap-rpms-tag", bootstrap.OutputTag) + assert.Equal(t, "azl4-dev-bootstrap-rpms-tag", bootstrap.ParentTag) + assert.Equal(t, ".azl4-dev~bootstrap", bootstrap.MockDistTag) + + rpms := names["azl4-dev-nvidia-rpms-target"] + assert.Equal(t, "azl4-dev-nvidia-rpms-tag", rpms.OutputTag) + assert.Equal(t, "azl4-dev-rpms-tag", rpms.ParentTag) + assert.Equal(t, ".azl4-dev", rpms.MockDistTag) +} + +func TestResolveTemplates_ExternalReposCopied(t *testing.T) { + config := &cttools.DistroConfig{ + Distros: map[string]cttools.Distro{ + "d": { + Description: "D", + Versions: map[string]cttools.Version{ + "1.0": { + Description: "v1.0", + ReleaseVer: "1.0", + EnvironmentPrefix: "p", + RPMMacroDist: ".p", + GitSourceRepos: map[string][]cttools.GitSourceRepo{ + "r": {{ + Ref: "https://example.com", + DefaultBranch: "main", + DefaultKojiRPMTarget: "p-rpms-target", + KojiTargets: "tmpl", + ParentPrefix: "p", + }}, + }, + }, + }, + }, + }, + KojiTargetsTemplates: map[string]map[string][]cttools.KojiTarget{ + "tmpl": { + "rpms-target": {{ + OutputTag: "rpms-tag", + BuildRoots: []cttools.BuildRootRef{{Type: "build", Value: "rpm"}}, + MockOptionsBase: "rpm", + ExternalRepos: []cttools.ExternalRepo{ + {Name: "fedora", URL: "https://fedora.example.com", MergeMode: "bare"}, + }, + }}, + }, + }, + MockOptionsTemplates: map[string]cttools.MockOptionsTemplate{ + "rpm": {Options: []string{"opt1"}}, + }, + BuildRootTemplates: map[string]cttools.BuildRootTemplate{ + "rpm": {Packages: []string{"bash"}}, + }, + } + + err := cttools.ResolveTemplates(config) + require.NoError(t, err) + + resolved := config.Distros["d"].Versions["1.0"].GitSourceRepos["r"][0].ResolvedKojiTargets + require.Len(t, resolved, 1) + require.Len(t, resolved[0].ExternalRepos, 1) + assert.Equal(t, "fedora", resolved[0].ExternalRepos[0].Name) +} + +func TestResolveTemplates_MissingTemplate(t *testing.T) { + config := &cttools.DistroConfig{ + Distros: map[string]cttools.Distro{ + "d": { + Description: "D", + Versions: map[string]cttools.Version{ + "1.0": { + Description: "v1.0", + ReleaseVer: "1.0", + EnvironmentPrefix: "p", + GitSourceRepos: map[string][]cttools.GitSourceRepo{ + "r": {{ + Ref: "https://example.com", + DefaultBranch: "main", + DefaultKojiRPMTarget: "p-rpms-target", + KojiTargets: "nonexistent", + ParentPrefix: "p", + }}, + }, + }, + }, + }, + }, + KojiTargetsTemplates: map[string]map[string][]cttools.KojiTarget{}, + } + + err := cttools.ResolveTemplates(config) + require.Error(t, err) + assert.Contains(t, err.Error(), "nonexistent") +} + +func TestFilterEnvironment_Found(t *testing.T) { + config := &cttools.DistroConfig{ + Environments: map[string]cttools.Environment{ + "ct-dev": {Resources: map[string][]map[string]any{"r1": {{"interface-type": "rpm-repo"}}}}, + "ct-prod": {Resources: map[string][]map[string]any{"r2": {{"interface-type": "image-gallery"}}}}, + }, + } + + err := cttools.FilterEnvironment(config, "ct-dev") + require.NoError(t, err) + + require.Len(t, config.Environments, 1) + require.Contains(t, config.Environments, "ct-dev") +} + +func TestFilterEnvironment_NotFound(t *testing.T) { + config := &cttools.DistroConfig{ + Environments: map[string]cttools.Environment{ + "ct-dev": {Resources: map[string][]map[string]any{}}, + }, + } + + err := cttools.FilterEnvironment(config, "ct-staging") + require.Error(t, err) + assert.Contains(t, err.Error(), "ct-staging") + assert.Contains(t, err.Error(), "not found") +} diff --git a/internal/app/azldev/core/cttools/types.go b/internal/app/azldev/core/cttools/types.go new file mode 100644 index 00000000..78d0a581 --- /dev/null +++ b/internal/app/azldev/core/cttools/types.go @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// Package cttools provides utilities for parsing and resolving Azure Linux distro configuration. +// +//nolint:tagliatelle // JSON output must use hyphenated keys to match the distro-config schema. +package cttools + +// DistroConfig is the top-level structure for the fully parsed/merged Azure Linux distro configuration. +type DistroConfig struct { + Distros map[string]Distro `toml:"distros" json:"distros,omitempty" yaml:"distros,omitempty"` + KojiTargetsTemplates map[string]map[string][]KojiTarget `toml:"koji-targets-templates" json:"koji-targets-templates,omitempty" yaml:"koji-targets-templates,omitempty"` + MockOptionsTemplates map[string]MockOptionsTemplate `toml:"mock-options-templates" json:"mock-options-templates,omitempty" yaml:"mock-options-templates,omitempty"` + BuildRootTemplates map[string]BuildRootTemplate `toml:"build-root-templates" json:"build-root-templates,omitempty" yaml:"build-root-templates,omitempty"` + Environments map[string]Environment `toml:"environments" json:"environments,omitempty" yaml:"environments,omitempty"` +} + +// Distro represents a distro definition (e.g. "azurelinux"). +type Distro struct { + Description string `toml:"description" json:"description" yaml:"description"` + ShadowAllowlists []ShadowAllowlist `toml:"shadow-allowlists" json:"shadow-allowlists,omitempty" yaml:"shadow-allowlists,omitempty"` + Versions map[string]Version `toml:"versions" json:"versions,omitempty" yaml:"versions,omitempty"` +} + +// ShadowAllowlist is a tag-name entry in a distro's shadow allowlist. +type ShadowAllowlist struct { + TagName string `toml:"tag-name" json:"tag-name" yaml:"tag-name"` +} + +// Version represents a distro version definition (e.g. "4.0-dev"). +type Version struct { + Description string `toml:"description" json:"description" yaml:"description"` + ReleaseVer string `toml:"release-ver" json:"release-ver" yaml:"release-ver"` + EnvironmentPrefix string `toml:"environment-prefix" json:"environment-prefix" yaml:"environment-prefix"` + RPMMacroDist string `toml:"rpm-macro-dist" json:"rpm-macro-dist" yaml:"rpm-macro-dist"` + RPMMacroDistBootstrap string `toml:"rpm-macro-dist-bootstrap" json:"rpm-macro-dist-bootstrap" yaml:"rpm-macro-dist-bootstrap"` + GitSourceRepos map[string][]GitSourceRepo `toml:"git-source-repos" json:"git-source-repos,omitempty" yaml:"git-source-repos,omitempty"` + BuildChannels map[string][]BuildChannel `toml:"build-channels" json:"build-channels,omitempty" yaml:"build-channels,omitempty"` + PublishChannels map[string][]PublishChannel `toml:"publish-channels" json:"publish-channels,omitempty" yaml:"publish-channels,omitempty"` +} + +// GitSourceRepo represents a git source repository definition within a distro version. +type GitSourceRepo struct { + Ref string `toml:"ref" json:"ref" yaml:"ref"` + DefaultBranch string `toml:"default-branch" json:"default-branch" yaml:"default-branch"` + DefaultKojiRPMTarget string `toml:"default-koji-rpms-target" json:"default-koji-rpms-target" yaml:"default-koji-rpms-target"` + KojiTargets string `toml:"koji-targets" json:"koji-targets" yaml:"koji-targets"` + RepoPrefix string `toml:"repo-prefix" json:"repo-prefix,omitempty" yaml:"repo-prefix,omitempty"` + ParentPrefix string `toml:"parent-prefix" json:"parent-prefix" yaml:"parent-prefix"` + ResolvedKojiTargets []ResolvedKojiTarget `toml:"resolved-koji-targets" json:"resolved-koji-targets,omitempty" yaml:"resolved-koji-targets,omitempty"` +} + +// ResolvedKojiTarget is a fully resolved koji target with all prefixes applied. +type ResolvedKojiTarget struct { + Name string `toml:"name" json:"name" yaml:"name"` + OutputTag string `toml:"output-tag" json:"output-tag" yaml:"output-tag"` + ParentTag string `toml:"parent-tag" json:"parent-tag,omitempty" yaml:"parent-tag,omitempty"` + BuildRoots []ResolvedBuildRoot `toml:"build-roots" json:"build-roots" yaml:"build-roots"` + MockOptions []string `toml:"mock-options" json:"mock-options" yaml:"mock-options"` + MockDistTag string `toml:"mock-dist-tag" json:"mock-dist-tag,omitempty" yaml:"mock-dist-tag,omitempty"` + ExternalRepos []ExternalRepo `toml:"external-repos" json:"external-repos,omitempty" yaml:"external-repos,omitempty"` +} + +// ResolvedBuildRoot is a build root entry with the template expanded to a package list. +type ResolvedBuildRoot struct { + Type string `toml:"type" json:"type" yaml:"type"` + Packages []string `toml:"packages" json:"packages" yaml:"packages"` +} + +// KojiTarget is a koji build target definition from a template. +type KojiTarget struct { + OutputTag string `toml:"output-tag" json:"output-tag" yaml:"output-tag"` + ParentTag string `toml:"parent-tag" json:"parent-tag,omitempty" yaml:"parent-tag,omitempty"` + BuildRoots []BuildRootRef `toml:"build-roots" json:"build-roots" yaml:"build-roots"` + MockOptionsBase string `toml:"mock-options-base" json:"mock-options-base" yaml:"mock-options-base"` + MockDistTag string `toml:"mock-dist-tag" json:"mock-dist-tag,omitempty" yaml:"mock-dist-tag,omitempty"` + ExternalRepos []ExternalRepo `toml:"external-repos" json:"external-repos,omitempty" yaml:"external-repos,omitempty"` +} + +// BuildRootRef references a build-root template by name. +type BuildRootRef struct { + Type string `toml:"type" json:"type" yaml:"type"` + Value string `toml:"value" json:"value" yaml:"value"` +} + +// ExternalRepo is an external repository definition on a koji target. +type ExternalRepo struct { + Name string `toml:"name" json:"name" yaml:"name"` + URL string `toml:"url" json:"url" yaml:"url"` + MergeMode string `toml:"merge-mode" json:"merge-mode" yaml:"merge-mode"` +} + +// MockOptionsTemplate defines a reusable set of mock/rpm options. +type MockOptionsTemplate struct { + Options []string `toml:"options" json:"options" yaml:"options"` +} + +// BuildRootTemplate defines a reusable package list for koji build roots. +type BuildRootTemplate struct { + Packages []string `toml:"packages" json:"packages" yaml:"packages"` +} + +// Environment is a Control Tower environment definition. +type Environment struct { + Resources map[string][]map[string]any `toml:"resources" json:"resources" yaml:"resources"` +} + +// BuildChannel specifies a koji target for routing builds. +type BuildChannel struct { + KojiTarget string `toml:"koji-target" json:"koji-target" yaml:"koji-target"` +} + +// PublishChannel specifies a resource for publishing artifacts. +type PublishChannel struct { + PublishResource string `toml:"publish-resource" json:"publish-resource" yaml:"publish-resource"` +} diff --git a/scenario/clismoke_test.go b/scenario/clismoke_test.go index 27764317..9cf390a6 100644 --- a/scenario/clismoke_test.go +++ b/scenario/clismoke_test.go @@ -6,6 +6,7 @@ package scenario_tests import ( + "encoding/json" "path/filepath" "strings" "testing" @@ -13,6 +14,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/scenario/internal/cmdtest" "github.com/microsoft/azure-linux-dev-tools/scenario/internal/snapshot" "github.com/microsoft/azure-linux-dev-tools/scenario/internal/testhelpers" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -163,3 +165,68 @@ excluded-paths = ['build/**', 'out/**'] snapshot.TestSnapshottableCmd(t, test) } + +// Tests that `azldev advanced ct-tools config-dump` parses, resolves, and outputs +// a fully merged distro configuration as valid JSON. +func TestCTToolsConfigDump(t *testing.T) { + t.Parallel() + + if testing.Short() { + t.Skip("skipping long test") + } + + // Use the self-contained test config under scenario/testdata/cttools/. + configPath, err := filepath.Abs("scenario/testdata/cttools/distro.toml") + require.NoError(t, err) + + test := cmdtest.NewScenarioTest( + "advanced", "ct-tools", "config-dump", + "--ct-config", configPath, + "--environment", "ct-test", + "-O", "json", + ).Locally() + + results, err := test.Run(t) + require.NoError(t, err) + require.Zero(t, results.ExitCode, "stderr: %s", results.Stderr) + + // Parse the JSON output and verify structure. + var config map[string]any + require.NoError(t, json.Unmarshal([]byte(results.Stdout), &config)) + + // Verify top-level keys. + assert.Contains(t, config, "distros") + assert.Contains(t, config, "koji-targets-templates") + assert.Contains(t, config, "mock-options-templates") + assert.Contains(t, config, "build-root-templates") + assert.Contains(t, config, "environments") + + // Verify distro was loaded. + distros, ok := config["distros"].(map[string]any) + require.True(t, ok) + assert.Contains(t, distros, "testdistro") + + // Verify environment was filtered to ct-test only. + envs, ok := config["environments"].(map[string]any) + require.True(t, ok) + assert.Len(t, envs, 1) + assert.Contains(t, envs, "ct-test") + + // Verify template resolution produced resolved-koji-targets. + td := distros["testdistro"].(map[string]any) + versions := td["versions"].(map[string]any) + v1 := versions["1.0-dev"].(map[string]any) + repos := v1["git-source-repos"].(map[string]any) + mainRepos := repos["main"].([]any) + mainRepo := mainRepos[0].(map[string]any) + + resolved, ok := mainRepo["resolved-koji-targets"].([]any) + require.True(t, ok) + assert.NotEmpty(t, resolved, "resolved-koji-targets should not be empty") + + // Verify a resolved target has the expected prefix. + first := resolved[0].(map[string]any) + name, ok := first["name"].(string) + require.True(t, ok) + assert.Contains(t, name, "td1-dev-") +} diff --git a/scenario/testdata/cttools/common.toml b/scenario/testdata/cttools/common.toml new file mode 100644 index 00000000..4b3ff5ff --- /dev/null +++ b/scenario/testdata/cttools/common.toml @@ -0,0 +1,4 @@ +include = [ + "templates/*.toml", + "resources/*.toml", +] diff --git a/scenario/testdata/cttools/distro.toml b/scenario/testdata/cttools/distro.toml new file mode 100644 index 00000000..8649781e --- /dev/null +++ b/scenario/testdata/cttools/distro.toml @@ -0,0 +1,7 @@ +include = [ + "versions/v1.toml", + "common.toml", +] + +[distros.testdistro] +description = "Test Distro" diff --git a/scenario/testdata/cttools/resources/ct-test.toml b/scenario/testdata/cttools/resources/ct-test.toml new file mode 100644 index 00000000..f6e18aa3 --- /dev/null +++ b/scenario/testdata/cttools/resources/ct-test.toml @@ -0,0 +1,12 @@ +[[environments.'ct-test'.resources.'dev-rpms-base']] +interface-type = "rpm-repo" +backend = "azure-blobstore-repo" +base-uri = "" +container-name = "$releasever/dev/base" + +[[environments.'ct-test'.resources.'dev-images']] +interface-type = "image-gallery" +backend = "azure-compute-gallery" +resource-name = "testGallery" +staging-blobstore-uri = "https://test.blob.core.windows.net/" +vm-definition-suffix = "-dev" diff --git a/scenario/testdata/cttools/templates/koji-targets.toml b/scenario/testdata/cttools/templates/koji-targets.toml new file mode 100644 index 00000000..1529c09b --- /dev/null +++ b/scenario/testdata/cttools/templates/koji-targets.toml @@ -0,0 +1,39 @@ +[[koji-targets-templates.base.'bootstrap-rpms-target']] +output-tag = "bootstrap-rpms-tag" +build-roots = [ + {"type" = "srpm-build", "value" = "srpm"}, + {"type" = "build", "value" = "rpm"}, +] +mock-options-base = "rpm" +mock-dist-tag = "rpm-macro-dist-bootstrap" +external-repos = [ + {"name" = "fedora-external", "url" = "https://fedora.example.com/$arch/os/", "merge-mode" = "bare"}, +] + +[[koji-targets-templates.base.'rpms-target']] +output-tag = "rpms-tag" +parent-tag = "bootstrap-rpms-tag" +build-roots = [ + {"type" = "srpm-build", "value" = "srpm"}, + {"type" = "build", "value" = "rpm"}, +] +mock-options-base = "rpm" +mock-dist-tag = "rpm-macro-dist" + +[[koji-targets-templates.prop.'bootstrap-rpms-target']] +output-tag = "bootstrap-rpms-tag" +parent-tag = "bootstrap-rpms-tag" +build-roots = [ + {"type" = "build", "value" = "rpm"}, +] +mock-options-base = "rpm" +mock-dist-tag = "rpm-macro-dist-bootstrap" + +[[koji-targets-templates.prop.'rpms-target']] +output-tag = "rpms-tag" +parent-tag = "rpms-tag" +build-roots = [ + {"type" = "build", "value" = "rpm"}, +] +mock-options-base = "rpm" +mock-dist-tag = "rpm-macro-dist" diff --git a/scenario/testdata/cttools/templates/options.toml b/scenario/testdata/cttools/templates/options.toml new file mode 100644 index 00000000..0843996e --- /dev/null +++ b/scenario/testdata/cttools/templates/options.toml @@ -0,0 +1,12 @@ +[mock-options-templates.rpm] +options = [ + "mock.isolation=simple", + "mock.package_manager=dnf5", + "rpm.macro.vendor=Test Corp", +] + +[build-root-templates.srpm] +packages = ["bash", "rpm-build"] + +[build-root-templates.rpm] +packages = ["bash", "gcc", "make"] diff --git a/scenario/testdata/cttools/versions/v1.toml b/scenario/testdata/cttools/versions/v1.toml new file mode 100644 index 00000000..c02dc5e1 --- /dev/null +++ b/scenario/testdata/cttools/versions/v1.toml @@ -0,0 +1,30 @@ +[distros.testdistro.versions.'1.0-dev'] +description = "Test Distro 1.0-dev" +release-ver = "1.0" +environment-prefix = "td1-dev" +rpm-macro-dist = ".td1-dev" +rpm-macro-dist-bootstrap = ".td1-dev~bootstrap" + +[[distros.testdistro.versions.'1.0-dev'.git-source-repos.'main']] +ref = "https://example.com/main.git" +default-branch = "main" +default-koji-rpms-target = "td1-dev-rpms-target" +koji-targets = "base" +parent-prefix = "td1-dev" + +[[distros.testdistro.versions.'1.0-dev'.git-source-repos.'proprietary']] +ref = "https://example.com/prop.git" +default-branch = "main" +default-koji-rpms-target = "td1-dev-prop-rpms-target" +koji-targets = "prop" +repo-prefix = "prop" +parent-prefix = "td1-dev" + +[[distros.testdistro.versions.'1.0-dev'.build-channels.'default-rpm']] +koji-target = "td1-dev-rpms-target" + +[[distros.testdistro.versions.'1.0-dev'.build-channels.'prop-rpm']] +koji-target = "td1-dev-prop-rpms-target" + +[[distros.testdistro.versions.'1.0-dev'.publish-channels.'rpm-base']] +publish-resource = "dev-rpms-base"