diff --git a/docs/user/reference/cli/azldev_component_render.md b/docs/user/reference/cli/azldev_component_render.md index a67f3a1..2da5f35 100644 --- a/docs/user/reference/cli/azldev_component_render.md +++ b/docs/user/reference/cli/azldev_component_render.md @@ -12,6 +12,9 @@ intended for check-in. The output directory is set via rendered-specs-dir in the project config, or via --output-dir on the command line. If neither is set, an error is returned. +Within the output directory, components are organized into letter-prefixed +subdirectories based on the first character of their name (e.g., specs/c/curl, +specs/v/vim). Unlike prepare-sources, render skips downloading source tarballs from the lookaside cache — only spec files, patches, scripts, and other git-tracked diff --git a/docs/user/reference/config/project.md b/docs/user/reference/config/project.md index d0e9610..eaea099 100644 --- a/docs/user/reference/config/project.md +++ b/docs/user/reference/config/project.md @@ -24,7 +24,7 @@ The `log-dir`, `work-dir`, `output-dir`, and `rendered-specs-dir` paths are reso - **`log-dir`** — build logs are written here (e.g., `azldev.log`) - **`work-dir`** — temporary per-component working directories are created under this path during builds (e.g., source preparation, SRPM construction) - **`output-dir`** — final build artifacts (RPMs, SRPMs) are placed here -- **`rendered-specs-dir`** — rendered spec and sidecar files are written here by `azldev component render` +- **`rendered-specs-dir`** — rendered spec and sidecar files are written here by `azldev component render`. Components are organized into letter-prefixed subdirectories (e.g., `SPECS/c/curl`, `SPECS/v/vim`) > **Note:** Do not edit files under these directories manually — they are managed by azldev and may be overwritten or cleaned at any time. diff --git a/internal/app/azldev/cmds/component/list_test.go b/internal/app/azldev/cmds/component/list_test.go index baba511..45f89b7 100644 --- a/internal/app/azldev/cmds/component/list_test.go +++ b/internal/app/azldev/cmds/component/list_test.go @@ -93,7 +93,7 @@ func TestListComponents_WithRenderedSpecsDir(t *testing.T) { result := results[0] assert.Equal(t, testComponentName, result.Name) - assert.Equal(t, filepath.Join(testRenderedDir, testComponentName), result.RenderedSpecDir) + assert.Equal(t, filepath.Join(testRenderedDir, "v", testComponentName), result.RenderedSpecDir) } func TestListComponents_MultipleWithRenderedSpecsDir(t *testing.T) { @@ -121,7 +121,14 @@ func TestListComponents_MultipleWithRenderedSpecsDir(t *testing.T) { require.NoError(t, err) require.Len(t, results, 2) + expectedDirs := map[string]string{ + "curl": filepath.Join(testRenderedDir, "c", "curl"), + "vim": filepath.Join(testRenderedDir, "v", "vim"), + } + for _, result := range results { - assert.Equal(t, filepath.Join(testRenderedDir, result.Name), result.RenderedSpecDir) + expected, ok := expectedDirs[result.Name] + require.True(t, ok, "unexpected component %q in results", result.Name) + assert.Equal(t, expected, result.RenderedSpecDir) } } diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index 2819b94..64e48c4 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -52,6 +52,9 @@ intended for check-in. The output directory is set via rendered-specs-dir in the project config, or via --output-dir on the command line. If neither is set, an error is returned. +Within the output directory, components are organized into letter-prefixed +subdirectories based on the first character of their name (e.g., specs/c/curl, +specs/v/vim). Unlike prepare-sources, render skips downloading source tarballs from the lookaside cache — only spec files, patches, scripts, and other git-tracked @@ -179,7 +182,7 @@ func RenderComponents(env *azldev.Env, options *RenderOptions) ([]*RenderResult, mockResultMap := batchMockProcess(env, mockProcessor, stagingDir, prepared) // ── Phase 3: Parallel finishing ── - parallelFinish(env, prepared, mockResultMap, results, stagingDir, options.OutputDir, + parallelFinish(env, prepared, mockResultMap, results, stagingDir, options.Force) // Clean up stale rendered directories when explicitly requested. @@ -511,7 +514,6 @@ func parallelFinish( mockResultMap map[string]*sources.ComponentMockResult, results []*RenderResult, stagingDir string, - outputDir string, allowOverwrite bool, ) { if len(prepared) == 0 { @@ -540,7 +542,7 @@ func parallelFinish( go func(prep *preparedComponent) { defer waitGroup.Done() - result := finishOneComponent(workerEnv, env, prep, mockResultMap, semaphore, stagingDir, outputDir, allowOverwrite) + result := finishOneComponent(workerEnv, env, prep, mockResultMap, semaphore, stagingDir, allowOverwrite) resultsChan <- finishResult{index: prep.index, result: result} }(prep) } @@ -568,7 +570,6 @@ func finishOneComponent( mockResultMap map[string]*sources.ComponentMockResult, semaphore chan struct{}, stagingDir string, - outputDir string, allowOverwrite bool, ) *RenderResult { componentName := prep.comp.GetName() @@ -593,7 +594,7 @@ func finishOneComponent( Status: renderStatusOK, } - err := finishComponentRender(env, prep, mockResultMap, stagingDir, outputDir, allowOverwrite) + err := finishComponentRender(env, prep, mockResultMap, stagingDir, allowOverwrite) if err != nil { slog.Error("Failed to finish rendering component", "component", componentName, "error", err) @@ -624,7 +625,6 @@ func finishComponentRender( prep *preparedComponent, mockResultMap map[string]*sources.ComponentMockResult, stagingDir string, - baseOutputDir string, allowOverwrite bool, ) error { componentName := prep.comp.GetName() @@ -658,12 +658,12 @@ func finishComponentRender( } // Copy rendered files to the component's output directory. - if copyErr := copyRenderedOutput(env, componentDir, baseOutputDir, componentName, allowOverwrite); copyErr != nil { + if copyErr := copyRenderedOutput(env, componentDir, prep.compOutputDir, allowOverwrite); copyErr != nil { return copyErr } slog.Info("Rendered component", "component", componentName, - "output", filepath.Join(baseOutputDir, componentName)) + "output", prep.compOutputDir) return nil } @@ -671,9 +671,7 @@ func finishComponentRender( // copyRenderedOutput copies the rendered files from tempDir to the component's output directory. // For managed output (inside project root), existing output is removed before copying. // For external output, existing directories cause an error. -func copyRenderedOutput(env *azldev.Env, tempDir, baseOutputDir, componentName string, allowOverwrite bool) error { - componentOutputDir := filepath.Join(baseOutputDir, componentName) - +func copyRenderedOutput(env *azldev.Env, tempDir, componentOutputDir string, allowOverwrite bool) error { exists, existsErr := fileutils.DirExists(env.FS(), componentOutputDir) if existsErr != nil { return fmt.Errorf("checking output directory %#q:\n%w", componentOutputDir, existsErr) @@ -704,7 +702,7 @@ func copyRenderedOutput(env *azldev.Env, tempDir, baseOutputDir, componentName s } if copyErr := fileutils.CopyDirRecursive(env, env.FS(), tempDir, componentOutputDir, copyOptions); copyErr != nil { - return fmt.Errorf("copying rendered files for %#q:\n%w", componentName, copyErr) + return fmt.Errorf("copying rendered files to %#q:\n%w", componentOutputDir, copyErr) } return nil @@ -770,6 +768,8 @@ func findSpecFile(fs opctx.FS, dir, componentName string) (string, error) { // cleanupStaleRenders removes rendered output directories for components that // no longer exist in the current configuration. Only called during full renders (-a). +// The output directory uses letter-prefix subdirectories (e.g., SPECS/c/curl), +// so this walks two levels: letter directories, then component directories within each. func cleanupStaleRenders(fs opctx.FS, currentComponents *components.ComponentSet, outputDir string) error { exists, existsErr := fileutils.Exists(fs, outputDir) if existsErr != nil { @@ -780,7 +780,7 @@ func cleanupStaleRenders(fs opctx.FS, currentComponents *components.ComponentSet return nil } - entries, err := fileutils.ReadDir(fs, outputDir) + letterEntries, err := fileutils.ReadDir(fs, outputDir) if err != nil { return fmt.Errorf("reading output directory %#q:\n%w", outputDir, err) } @@ -791,22 +791,34 @@ func cleanupStaleRenders(fs opctx.FS, currentComponents *components.ComponentSet currentNames[comp.GetName()] = true } - for _, entry := range entries { - // Skip non-directories and known non-component files. - if !entry.IsDir() { + for _, letterEntry := range letterEntries { + if !letterEntry.IsDir() { continue } - if currentNames[entry.Name()] { - continue + letterDir := filepath.Join(outputDir, letterEntry.Name()) + + compEntries, readErr := fileutils.ReadDir(fs, letterDir) + if readErr != nil { + return fmt.Errorf("reading letter directory %#q:\n%w", letterDir, readErr) } - stalePath := filepath.Join(outputDir, entry.Name()) + for _, compEntry := range compEntries { + if !compEntry.IsDir() { + continue + } + + if currentNames[compEntry.Name()] { + continue + } - slog.Info("Removing stale rendered output", "directory", stalePath) + stalePath := filepath.Join(letterDir, compEntry.Name()) - if removeErr := fs.RemoveAll(stalePath); removeErr != nil { - return fmt.Errorf("removing stale directory %#q:\n%w", stalePath, removeErr) + slog.Info("Removing stale rendered output", "directory", stalePath) + + if removeErr := fs.RemoveAll(stalePath); removeErr != nil { + return fmt.Errorf("removing stale directory %#q:\n%w", stalePath, removeErr) + } } } diff --git a/internal/app/azldev/cmds/component/render_internal_test.go b/internal/app/azldev/cmds/component/render_internal_test.go index b46e1d4..85f7d42 100644 --- a/internal/app/azldev/cmds/component/render_internal_test.go +++ b/internal/app/azldev/cmds/component/render_internal_test.go @@ -86,11 +86,13 @@ func TestCleanupStaleRenders(t *testing.T) { testFS := afero.NewMemMapFs() ctrl := gomock.NewController(t) - // Create output directories for curl, wget, and stale-pkg. + // Create letter-prefixed output directories for curl, wget, and stale-pkg. for _, name := range []string{"curl", "wget", "stale-pkg"} { - require.NoError(t, fileutils.MkdirAll(testFS, filepath.Join("/output", name))) + prefix := string(name[0]) + dir := filepath.Join("/output", prefix, name) + require.NoError(t, fileutils.MkdirAll(testFS, dir)) require.NoError(t, fileutils.WriteFile(testFS, - filepath.Join("/output", name, name+".spec"), + filepath.Join(dir, name+".spec"), []byte("Name: "+name), fileperms.PublicFile)) } @@ -108,13 +110,14 @@ func TestCleanupStaleRenders(t *testing.T) { // curl and wget should still exist. for _, name := range []string{"curl", "wget"} { - exists, existsErr := fileutils.Exists(testFS, filepath.Join("/output", name)) + prefix := string(name[0]) + exists, existsErr := fileutils.Exists(testFS, filepath.Join("/output", prefix, name)) require.NoError(t, existsErr) assert.True(t, exists, "%s should still exist", name) } // stale-pkg should be removed. - exists, err := fileutils.Exists(testFS, "/output/stale-pkg") + exists, err := fileutils.Exists(testFS, "/output/s/stale-pkg") require.NoError(t, err) assert.False(t, exists, "stale-pkg should be removed") }) diff --git a/internal/app/azldev/core/components/renderedspecdir.go b/internal/app/azldev/core/components/renderedspecdir.go index 0b6501e..e40c4af 100644 --- a/internal/app/azldev/core/components/renderedspecdir.go +++ b/internal/app/azldev/core/components/renderedspecdir.go @@ -6,12 +6,14 @@ package components import ( "fmt" "path/filepath" + "strings" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" ) // RenderedSpecDir returns the rendered spec output directory for a given component. -// The path is computed as {renderedSpecsDir}/{componentName}. +// Components are organized by the lowercase first letter of their name: +// {renderedSpecsDir}/{letter}/{componentName} (e.g., "SPECS/c/curl"). // Returns an empty string if renderedSpecsDir is not configured (empty). // Returns an error if componentName is unsafe (absolute, contains path separators // or traversal sequences). @@ -24,5 +26,7 @@ func RenderedSpecDir(renderedSpecsDir, componentName string) (string, error) { return "", nil } - return filepath.Join(renderedSpecsDir, componentName), nil + prefix := strings.ToLower(componentName[:1]) + + return filepath.Join(renderedSpecsDir, prefix, componentName), nil } diff --git a/internal/app/azldev/core/components/renderedspecdir_test.go b/internal/app/azldev/core/components/renderedspecdir_test.go index 690baec..11a45af 100644 --- a/internal/app/azldev/core/components/renderedspecdir_test.go +++ b/internal/app/azldev/core/components/renderedspecdir_test.go @@ -12,10 +12,10 @@ import ( ) func TestRenderedSpecDir(t *testing.T) { - t.Run("ReturnsPathWhenConfigured", func(t *testing.T) { + t.Run("ReturnsLetterPrefixedPath", func(t *testing.T) { result, err := components.RenderedSpecDir("/path/to/specs", "vim") require.NoError(t, err) - assert.Equal(t, "/path/to/specs/vim", result) + assert.Equal(t, "/path/to/specs/v/vim", result) }) t.Run("ReturnsEmptyWhenNotConfigured", func(t *testing.T) { @@ -24,10 +24,16 @@ func TestRenderedSpecDir(t *testing.T) { assert.Empty(t, result) }) + t.Run("LowercasesPrefixForUppercaseName", func(t *testing.T) { + result, err := components.RenderedSpecDir("/specs", "SymCrypt") + require.NoError(t, err) + assert.Equal(t, "/specs/s/SymCrypt", result) + }) + t.Run("HandlesComponentNameWithDashes", func(t *testing.T) { result, err := components.RenderedSpecDir("/rendered", "my-component") require.NoError(t, err) - assert.Equal(t, "/rendered/my-component", result) + assert.Equal(t, "/rendered/m/my-component", result) }) t.Run("RejectsAbsoluteComponentName", func(t *testing.T) { diff --git a/internal/utils/fileutils/file.go b/internal/utils/fileutils/file.go index 53fa682..680969e 100644 --- a/internal/utils/fileutils/file.go +++ b/internal/utils/fileutils/file.go @@ -96,5 +96,12 @@ func ValidateFilename(filename string) error { return fmt.Errorf("filename %#q must not contain backslashes", filename) } + // Reject non-ASCII characters. RPM package names are ASCII-only, and + // non-ASCII bytes would produce garbled single-byte prefixes when used + // for letter-bucketed directory layouts. + if strings.ContainsFunc(filename, func(r rune) bool { return r > unicode.MaxASCII }) { + return fmt.Errorf("filename %#q must contain only ASCII characters", filename) + } + return nil } diff --git a/internal/utils/fileutils/file_test.go b/internal/utils/fileutils/file_test.go index 5641229..b27baf3 100644 --- a/internal/utils/fileutils/file_test.go +++ b/internal/utils/fileutils/file_test.go @@ -92,6 +92,7 @@ func TestValidateFilename(t *testing.T) { {name: "tab in name", filename: "has\ttab.tar.gz", expectedError: "must not contain whitespace"}, {name: "null byte in name", filename: "has\x00null.tar.gz", expectedError: "must not contain null bytes"}, {name: "backslash in name", filename: "foo\\bar.tar.gz", expectedError: "must not contain backslashes"}, + {name: "non-ASCII characters", filename: "foo\x80bar.tar.gz", expectedError: "must contain only ASCII characters"}, } for _, tc := range tests { diff --git a/scenario/component_render_test.go b/scenario/component_render_test.go index f043475..4e05c6f 100644 --- a/scenario/component_render_test.go +++ b/scenario/component_render_test.go @@ -63,7 +63,7 @@ func TestRenderSimpleLocalSpec(t *testing.T) { "Simple spec should render without warnings when rpmautospec is installed") // Verify rendered spec file exists with expected content. - renderedSpecPath := results.GetProjectOutputPath("SPECS", "test-render", "test-render.spec") + renderedSpecPath := results.GetProjectOutputPath("SPECS", "t", "test-render", "test-render.spec") require.FileExists(t, renderedSpecPath) content, err := os.ReadFile(renderedSpecPath) @@ -113,7 +113,7 @@ func TestRenderWithConfiguredOutputDir(t *testing.T) { assert.Equal(t, "ok", output[0]["status"], "Spec should render ok with config-provided output dir") - renderedSpecPath := results.GetProjectOutputPath("SPECS", "config-test", "config-test.spec") + renderedSpecPath := results.GetProjectOutputPath("SPECS", "c", "config-test", "config-test.spec") require.FileExists(t, renderedSpecPath) content, err := os.ReadFile(renderedSpecPath) @@ -161,7 +161,7 @@ func TestRenderWithOverlayApplied(t *testing.T) { assert.Equal(t, "ok", output[0]["status"], "Spec should render as ok when rpmautospec is installed") // Verify the overlay was applied — the rendered spec should contain the added tag. - renderedSpecPath := results.GetProjectOutputPath("SPECS", "test-overlay", "test-overlay.spec") + renderedSpecPath := results.GetProjectOutputPath("SPECS", "t", "test-overlay", "test-overlay.spec") require.FileExists(t, renderedSpecPath) content, err := os.ReadFile(renderedSpecPath) @@ -218,11 +218,11 @@ func TestRenderWithPatchSidecar(t *testing.T) { assert.Equal(t, "ok", output[0]["status"], "Spec should render as ok when rpmautospec is installed") // Verify the patch file is in the rendered output. - patchPath := results.GetProjectOutputPath("SPECS", "test-patch", "fix-stuff.patch") + patchPath := results.GetProjectOutputPath("SPECS", "t", "test-patch", "fix-stuff.patch") require.FileExists(t, patchPath, "Patch sidecar should be in rendered output") // Verify the spec references the patch. - renderedSpecPath := results.GetProjectOutputPath("SPECS", "test-patch", "test-patch.spec") + renderedSpecPath := results.GetProjectOutputPath("SPECS", "t", "test-patch", "test-patch.spec") content, err := os.ReadFile(renderedSpecPath) require.NoError(t, err) @@ -249,7 +249,7 @@ func TestRenderStaleCleanup(t *testing.T) { projecttest.AddComponent(localComponentConfig("keep-me")), projecttest.UseTestDefaultConfigs(), projecttest.WithGitRepo(), - projecttest.AddFile("SPECS/stale-component/RENDER_FAILED", "Rendering failed.\n"), + projecttest.AddFile("SPECS/s/stale-component/RENDER_FAILED", "Rendering failed.\n"), ) results := projecttest.NewProjectTest( @@ -265,11 +265,11 @@ func TestRenderStaleCleanup(t *testing.T) { assert.Equal(t, "keep-me", output[0]["component"]) // Verify the stale directory was cleaned up. - stalePath := results.GetProjectOutputPath("SPECS", "stale-component") + stalePath := results.GetProjectOutputPath("SPECS", "s", "stale-component") assert.NoDirExists(t, stalePath, "Stale component directory should have been removed") // Verify the kept component still exists. - keptPath := results.GetProjectOutputPath("SPECS", "keep-me") + keptPath := results.GetProjectOutputPath("SPECS", "k", "keep-me") assert.DirExists(t, keptPath, "Rendered component directory should still exist") } @@ -295,7 +295,7 @@ func TestRenderRefusesOverwriteWithoutForce(t *testing.T) { projecttest.AddComponent(localComponentConfig("no-clobber")), projecttest.UseTestDefaultConfigs(), projecttest.WithGitRepo(), - projecttest.AddFile("SPECS/no-clobber/existing-file.txt", "do not delete me\n"), + projecttest.AddFile("SPECS/n/no-clobber/existing-file.txt", "do not delete me\n"), ) results := projecttest.NewProjectTest( @@ -313,7 +313,7 @@ func TestRenderRefusesOverwriteWithoutForce(t *testing.T) { "Render should fail when output dir exists without --force") // The pre-existing file should NOT have been deleted. - existingPath := results.GetProjectOutputPath("SPECS", "no-clobber", "existing-file.txt") + existingPath := results.GetProjectOutputPath("SPECS", "n", "no-clobber", "existing-file.txt") require.FileExists(t, existingPath, "Pre-existing file should be preserved when --force is not set") } @@ -389,7 +389,7 @@ License: MIT "Spec with golang macros should render ok via mock processing") // The spec file should exist in the output. - renderedSpecPath := results.GetProjectOutputPath("SPECS", "golang-example", "golang-example.spec") + renderedSpecPath := results.GetProjectOutputPath("SPECS", "g", "golang-example", "golang-example.spec") require.FileExists(t, renderedSpecPath, "Spec should be rendered via mock processing") @@ -465,10 +465,10 @@ func TestRenderMultipleComponentsParallel(t *testing.T) { "comp-beta should render ok") // Verify both rendered specs exist. - specAlphaPath := results.GetProjectOutputPath("SPECS", "comp-alpha", "comp-alpha.spec") + specAlphaPath := results.GetProjectOutputPath("SPECS", "c", "comp-alpha", "comp-alpha.spec") require.FileExists(t, specAlphaPath) - specBetaPath := results.GetProjectOutputPath("SPECS", "comp-beta", "comp-beta.spec") + specBetaPath := results.GetProjectOutputPath("SPECS", "c", "comp-beta", "comp-beta.spec") require.FileExists(t, specBetaPath) // comp-beta uses %autorelease, so rpmautospec should have processed it. @@ -529,7 +529,7 @@ func TestRenderBrokenSpecWithGoodSpec(t *testing.T) { assert.Equal(t, "ok", resultMap["good-pkg"]["status"], "good-pkg should render ok even when another component fails") - goodSpecPath := results.GetProjectOutputPath("SPECS", "good-pkg", "good-pkg.spec") + goodSpecPath := results.GetProjectOutputPath("SPECS", "g", "good-pkg", "good-pkg.spec") require.FileExists(t, goodSpecPath) // Broken spec should produce an error status. @@ -537,6 +537,6 @@ func TestRenderBrokenSpecWithGoodSpec(t *testing.T) { "broken-pkg should report error for malformed spec") // Error marker file should be written for the broken component. - markerPath := results.GetProjectOutputPath("SPECS", "broken-pkg", "RENDER_FAILED") + markerPath := results.GetProjectOutputPath("SPECS", "b", "broken-pkg", "RENDER_FAILED") require.FileExists(t, markerPath, "RENDER_FAILED marker should exist for broken component") }