Make adding a language mean adding a file

CONTRIBUTING.md

@@ -37,7 +37,8 @@ dotnet run --project src/SignsOfAI.Cli -- check some-file.md
 
 ## Adding or changing a rule
 
-Rules live in two JSON files: `src/SignsOfAI.Core/Rules/Packs/rules.en.json` and `rules.es.json`.
+Rules live in JSON files under `src/SignsOfAI.Core/Rules/Packs/` — today `rules.en.json` and
+`rules.es.json`, and any `rules.<code>.json` you add (see *Adding a whole language* below).
 There are two kinds.
 
 **Lexical** — single overused words. All inflections go in `terms`:
@@ -120,6 +121,29 @@ into it will be asked for rework. Spanish AI writing has its own tells (*sumérg
 de*, *cabe destacar que*, *un rico tapiz de*). If you propose a Spanish rule, ground it in Spanish
 text you have actually seen a model produce.
 
+### Adding a whole language
+
+**Drop `rules.<code>.json` into `src/SignsOfAI.Core/Rules/Packs/` and it is picked up.** No project
+file to edit, no C# to touch, no list to register in — the build embeds the pack by wildcard and
+`RulePackLoader` finds it by name. That is deliberate: a contributor who cannot get their language
+heard without editing a build script will not contribute a language.
+
+Derive it, do not translate it, for the reason above.
+
+Two things the tool will then say on your behalf, and you should expect both:
+
+- **Until your pack exists, text in your language is examined with the English catalog**, and the
+  report says so and tells the reader to treat the score as meaningless — a low number would mean
+  nothing was looked for, not that nothing was found.
+- **A new language has no measured error rate.** The calibration corpus contains no texts in it, so
+  no threshold is supported and no verdict is printed. That is honest rather than broken, and it is
+  fixed by contributing texts published before 2022 — see `Docs/Calibration/README.md`. **Roughly
+  seventy-five such texts are worth more to your language's users than the rule pack is**, because
+  they are what lets the tool say anything at all about how often it is wrong there.
+
+The interface (`wwwroot/i18n/<code>.json`) and the report prose (`Reporting/report.<code>.json`) are
+separate and can land in separate pull requests: see [`Docs/TRANSLATING.md`](Docs/TRANSLATING.md).
+
 ## Tests
 
 Every rule change needs a test in `tests/SignsOfAI.Core.Tests`. The pattern is short:

src/SignsOfAI.Core/AiWritingAnalyzer.cs

@@ -89,6 +89,7 @@ public AnalysisResult Analyze(string text, string? language = null, IReadOnlyLis
         return new AnalysisResult
         {
             Language = lang,
+            RulePackLanguage = RulePackLoader.Resolve(lang).Language,
             Findings = findings,
             CategoryScores = byCategory,
             OverallScore = overall,

src/SignsOfAI.Core/Model/AnalysisResult.cs

@@ -9,9 +9,20 @@ public sealed record CategoryScore(SignCategory Category, double Score, int Find
 /// <summary>The complete result of analyzing a document.</summary>
 public sealed record AnalysisResult
 {
-    /// <summary>Language code actually used for analysis ("en" or "es").</summary>
+    /// <summary>The language of the text: given by the caller, or detected.</summary>
     public required string Language { get; init; }
 
+    /// <summary>
+    /// The language whose rule pack actually supplied the tells. Equal to <see cref="Language"/>
+    /// except when that language has no pack yet, in which case the English catalog was used.
+    ///
+    /// The two must not be conflated. Running English rules over French prose finds few tells, and
+    /// reporting that as a French analysis would present "nothing fired" as a result when nothing
+    /// French was ever looked for. Rule packs are files anyone can contribute, so a language without
+    /// one is an ordinary state that hosts should describe rather than an error.
+    /// </summary>
+    public string RulePackLanguage { get; init; } = "";
+
     /// <summary>
     /// Everything that matched, ordered by position in the text — both the findings that count as
     /// evidence and the ones the writer is using at a rate people write at. Highlighting works from

src/SignsOfAI.Core/Reporting/EvidenceReport.cs

@@ -119,6 +119,16 @@ public static string ToMarkdown(AnalysisResult result, ReportOptions? options =
             LanguageName(text, result.Language), result.Statistics.WordCount,
             result.Statistics.SentenceCount, Num(result.Statistics.Burstiness, 2));
         sb.AppendLine();
+
+        // Said before the error rate, because it outranks it: a rate measured on English writing
+        // describes nothing about what these rules do to French prose they were never written for.
+        if (result.RulePackLanguage is { Length: > 0 } packLanguage
+            && !packLanguage.Equals(result.Language, StringComparison.OrdinalIgnoreCase))
+        {
+            AppendBlock(sb, text, ReportMessages.NoRulePack, LanguageName(text, result.Language));
+            sb.AppendLine();
+        }
+
         AppendLocalized(sb, text, Caveat(text, result.Language));
         sb.AppendLine();
 

src/SignsOfAI.Core/Reporting/ReportMessages.cs

@@ -16,6 +16,7 @@ public static class ReportMessages
     public const string FallbackMarker = "fallback.marker";
     public const string FallbackSummary = "fallback.summary";
     public const string FallbackLanguage = "fallback.language";
+    public const string NoRulePack = "analysis.no-rule-pack";
     public const string DefaultTitle = "default.title";
     public const string MetaDocument = "meta.document";
     public const string MetaGenerated = "meta.generated";
@@ -93,6 +94,7 @@ public static class ReportMessages
         [FallbackMarker] = 0,
         [FallbackSummary] = 1,
         [FallbackLanguage] = 1,
+        [NoRulePack] = 1,          // {0} the language of the text
         [DefaultTitle] = 0,
         [MetaDocument] = 1,
         [MetaGenerated] = 2,
@@ -176,6 +178,11 @@ public static class ReportMessages
         [FallbackLanguage] = "This report is not available in {0}, so the whole of it is shown in English. " +
                              "Nothing has been withheld or shortened, but a reader who cannot read English " +
                              "cannot read the part that limits the score, and that part is the point of the page.",
+        [NoRulePack] = "> **There is no rule pack for {0} yet, so this text was examined with the English one.** " +
+                       "Treat the score as saying nothing at all: the tells this tool knows are English " +
+                       "ones, and few of them can fire on writing in another language — so a low number " +
+                       "here means nothing was looked for, not that nothing was found. Rule packs are " +
+                       "JSON files anyone can contribute.",
         [DefaultTitle] = "Writing analysis report",
         [MetaDocument] = "**Document:** {0}",
         [MetaGenerated] = "**Generated:** {0} · **Engine:** SignsOfAI {1}",

src/SignsOfAI.Core/Reporting/report.en.json

@@ -5,6 +5,7 @@
     "fallback.marker": { "text": "This block has not been translated yet; it is shown in English." },
     "fallback.summary": { "text": "This report contains {0} block(s) not yet translated. Each is marked and shown in English." },
     "fallback.language": { "text": "This report is not available in {0}, so the whole of it is shown in English. Nothing has been withheld or shortened, but a reader who cannot read English cannot read the part that limits the score, and that part is the point of the page." },
+    "analysis.no-rule-pack": { "text": "> **There is no rule pack for {0} yet, so this text was examined with the English one.** Treat the score as saying nothing at all: the tells this tool knows are English ones, and few of them can fire on writing in another language — so a low number here means nothing was looked for, not that nothing was found. Rule packs are JSON files anyone can contribute." },
     "default.title": { "text": "Writing analysis report" },
     "meta.document": { "text": "**Document:** {0}" },
     "meta.generated": { "text": "**Generated:** {0} · **Engine:** SignsOfAI {1}" },

src/SignsOfAI.Core/Reporting/report.es.json

@@ -14,6 +14,10 @@
       "text": "Este informe no está disponible en {0}, así que se muestra completo en inglés. No se ha ocultado ni acortado nada, pero quien no lea inglés no puede leer la parte que limita la puntuación, y esa parte es la razón de ser de esta página.",
       "sourceHash": "c43be4647f39ade1600db2c60ed02ecd38d3539b195ec9afaae737316234a6d4"
     },
+    "analysis.no-rule-pack": {
+      "text": "> **Todavía no existe un catálogo de reglas para {0}, así que este texto se examinó con el de inglés.** Trate la puntuación como si no dijera absolutamente nada: las señales que conoce esta herramienta son inglesas, y pocas pueden activarse sobre escritura en otro idioma — de modo que un número bajo aquí significa que no se buscó nada, no que no se encontró nada. Los catálogos de reglas son archivos JSON que cualquiera puede aportar.",
+      "sourceHash": "148085719a308a40ec90895e2bd4621ed8367db5573f5d9d31f94d4c99e5e344"
+    },
     "default.title": {
       "text": "Informe del análisis de escritura",
       "sourceHash": "90b8ccc0903d87a2f8ba07531f1e76736d5fce4adab65154f05ac147b919b291"

src/SignsOfAI.Core/Rules/RulePackLoader.cs

@@ -9,21 +9,54 @@ public static class RulePackLoader
 {
     private static readonly ConcurrentDictionary<string, RulePack> Cache = new();
 
-    /// <summary>Loads the rule-pack for a language code ("en"/"es"), falling back to English.</summary>
-    public static RulePack Load(string language)
+    /// <summary>Loads the rule-pack for a language code, falling back to English.</summary>
+    public static RulePack Load(string language) => Resolve(language).Pack;
+
+    /// <summary>
+    /// The pack, and the language it was actually built for.
+    ///
+    /// These differ whenever a language has no pack yet, and the difference has to travel: a text
+    /// analysed with the English catalog is not a French analysis, and a result that claimed to be
+    /// one would be saying nothing fired in French when nothing French was ever looked for. Rule
+    /// packs are files anyone can add, so this is the ordinary case for a new language rather than
+    /// an error.
+    /// </summary>
+    public static (RulePack Pack, string Language) Resolve(string? language)
     {
         var lang = string.IsNullOrWhiteSpace(language) ? "en" : language.ToLowerInvariant();
-        return Cache.GetOrAdd(lang, LoadFromResource);
+        return (Cache.GetOrAdd(lang, LoadFromResource), Available(lang) ? lang : "en");
     }
 
+    /// <summary>Whether a built-in pack exists for this language. Adding one is adding a file.</summary>
+    public static bool Available(string? language) =>
+        !string.IsNullOrWhiteSpace(language)
+        && typeof(RulePackLoader).Assembly.GetManifestResourceInfo(ResourceName(language)) is not null;
+
+    /// <summary>
+    /// Every language with a built-in pack, so hosts can offer what exists rather than a hardcoded
+    /// pair that a contributor cannot extend.
+    /// </summary>
+    public static IReadOnlyList<string> Languages { get; } =
+        [.. typeof(RulePackLoader).Assembly.GetManifestResourceNames()
+            .Where(n => n.StartsWith(Prefix, StringComparison.Ordinal)
+                        && n.EndsWith(".json", StringComparison.Ordinal))
+            .Select(n => n[Prefix.Length..^".json".Length])
+            .Where(n => n.Length is > 0 and <= 12 && n.All(char.IsAsciiLetterLower))
+            .Order(StringComparer.Ordinal)];
+
+    private const string Prefix = "SignsOfAI.Core.Rules.Packs.rules.";
+
+    private static string ResourceName(string language) =>
+        $"SignsOfAI.Core.Rules.Packs.rules.{language.ToLowerInvariant()}.json";
+
     private static RulePack LoadFromResource(string language)
     {
         var asm = typeof(RulePackLoader).Assembly;
-        var resourceName = $"SignsOfAI.Core.Rules.Packs.rules.{language}.json";
+        var resourceName = ResourceName(language);
 
         using var stream = asm.GetManifestResourceStream(resourceName)
             ?? (language != "en"
-                ? asm.GetManifestResourceStream("SignsOfAI.Core.Rules.Packs.rules.en.json")
+                ? asm.GetManifestResourceStream(ResourceName("en"))
                 : null)
             ?? throw new InvalidOperationException(
                 $"Rule-pack resource '{resourceName}' not found. Available: " +

src/SignsOfAI.Core/SignsOfAI.Core.csproj

@@ -26,11 +26,16 @@
   </ItemGroup>
 
   <ItemGroup>
-    <!-- WithCulture=false is essential: filenames like rules.en.json / rules.es.json would
+    <!-- A wildcard, so adding a language is adding a file. Listing packs one by one made every
+         translator edit the build to be heard, which contradicts this project's own rule that
+         extension points are JSON anyone can send by pull request and never compiled code.
+
+         WithCulture=false is essential: filenames like rules.en.json / rules.es.json would
          otherwise be mistaken for culture-specific satellite resources and dropped from the
          main assembly manifest. LogicalName pins the reflection lookup name. -->
-    <EmbeddedResource Include="Rules\Packs\rules.en.json" WithCulture="false" LogicalName="SignsOfAI.Core.Rules.Packs.rules.en.json" />
-    <EmbeddedResource Include="Rules\Packs\rules.es.json" WithCulture="false" LogicalName="SignsOfAI.Core.Rules.Packs.rules.es.json" />
+    <EmbeddedResource Include="Rules\Packs\rules.*.json" WithCulture="false">
+      <LogicalName>SignsOfAI.Core.Rules.Packs.%(Filename)%(Extension)</LogicalName>
+    </EmbeddedResource>
 
     <!-- Reader-facing report prose is selected by interface language, independently from the rule
          pack selected for the analysed text. LogicalName keeps dotted locale filenames in the main

tests/SignsOfAI.Core.Tests/RulePackLoaderTests.cs

@@ -0,0 +1,93 @@
+using SignsOfAI.Core;
+using SignsOfAI.Core.Reporting;
+using SignsOfAI.Core.Rules;
+
+namespace SignsOfAI.Core.Tests;
+
+/// <summary>
+/// The project tells contributors that adding a language is adding a file. These are the tests that
+/// make that true rather than aspirational — and that stop a language without a pack being reported
+/// as though it had one.
+/// </summary>
+public class RulePackLoaderTests
+{
+    [Fact]
+    public void Every_pack_file_in_the_project_is_discoverable_without_a_build_edit()
+    {
+        // The packs were listed one by one in the .csproj, so a translator had to edit the build to
+        // be heard at all. A wildcard replaced that; this test is what keeps it a wildcard.
+        Assert.Contains("en", RulePackLoader.Languages);
+        Assert.Contains("es", RulePackLoader.Languages);
+
+        var onDisk = Directory
+            .GetFiles(ProjectPacksDirectory(), "rules.*.json")
+            .Select(f => Path.GetFileNameWithoutExtension(f)!.Split('.')[1])
+            .Order(StringComparer.Ordinal);
+
+        Assert.Equal(onDisk, RulePackLoader.Languages.Order(StringComparer.Ordinal));
+    }
+
+    [Fact]
+    public void A_language_with_no_pack_says_which_pack_it_actually_used()
+    {
+        // Silently loading English while the result claimed the requested language turned "nothing
+        // fired" into a finding, when nothing had been looked for.
+        //
+        // "zz" throughout, never a real code: this suite must keep passing on the day somebody
+        // contributes the language it uses as its example, and picking "fr" would make a welcome
+        // pull request look like a regression.
+        var (_, language) = RulePackLoader.Resolve("zz");
+
+        Assert.Equal("en", language);
+        Assert.False(RulePackLoader.Available("zz"));
+    }
+
+    [Fact]
+    public void A_language_with_a_pack_reports_itself()
+    {
+        Assert.Equal("es", RulePackLoader.Resolve("es").Language);
+        Assert.True(RulePackLoader.Available("es"));
+    }
+
+    [Fact]
+    public void The_result_keeps_the_two_languages_apart()
+    {
+        var result = new AiWritingAnalyzer().Analyze("Le texte est court mais suffisant.", "zz");
+
+        // The text is in the language asked for. The rules that read it were not.
+        Assert.Equal("zz", result.Language);
+        Assert.Equal("en", result.RulePackLanguage);
+    }
+
+    [Fact]
+    public void The_report_refuses_to_present_an_English_reading_as_a_result_in_that_language()
+    {
+        var result = new AiWritingAnalyzer().Analyze(
+            "La rédaction académique exige de la précision et une structure claire.", "zz");
+
+        var report = EvidenceReport.ToMarkdown(result);
+
+        Assert.Contains("no rule pack for", report);
+        Assert.Contains("nothing was looked for", report);
+    }
+
+    [Fact]
+    public void A_language_that_has_a_pack_carries_no_such_warning()
+    {
+        var result = new AiWritingAnalyzer().Analyze(
+            "La redacción académica exige precisión y una estructura clara.", "es");
+
+        Assert.DoesNotContain("no rule pack for", EvidenceReport.ToMarkdown(result));
+    }
+
+    /// <summary>The packs as they sit in the repository, not as the build happened to embed them.</summary>
+    private static string ProjectPacksDirectory()
+    {
+        var dir = new DirectoryInfo(AppContext.BaseDirectory);
+        while (dir is not null && !File.Exists(Path.Combine(dir.FullName, "SignsOfAI.slnx")))
+            dir = dir.Parent;
+
+        Assert.NotNull(dir);
+        return Path.Combine(dir!.FullName, "src", "SignsOfAI.Core", "Rules", "Packs");
+    }
+}