diff --git a/Docs/CALIBRATION.md b/Docs/CALIBRATION.md index 2d76569..f44e709 100644 --- a/Docs/CALIBRATION.md +++ b/Docs/CALIBRATION.md @@ -11,7 +11,7 @@ It is **not an accuracy figure**. Accuracy needs machine-written text to measure - **Corpus** `signsofai-human-baseline`, fingerprint `123fa5b9ebca3f29` - **Texts** 90 (280,221 words) - **Engine** SignsOfAI.Core 0.4.0 -- **Run** 2026-08-05 +- **Run** 2026-08-24 - **Target false-positive rate** 5% Every text here was published before generative models could have written it. That is the whole basis for calling it human, and it is a stronger guarantee than any classifier offers about anything. The manifest names each source, its licence and its year, so the claim can be traced rather than trusted. diff --git a/README.md b/README.md index d1a1c97..26167d5 100644 --- a/README.md +++ b/README.md @@ -384,3 +384,9 @@ The optional server is a normal ASP.NET Core app (`dotnet publish` the `SignsOfA Created by **Pedro Hernández — PeopleWorks**, [Microsoft MVP for .NET](https://mvp.microsoft.com/en-US/mvp/profile/24060a02-dbc6-44ec-bca5-c213ff9835c5). Detection markers are grounded in linguistics research on AI stylometry — see `Docs/GoogleResearch.md`. + +The `chat.*` rules — the assistant's own turn, left in the document — were adapted from the pattern +set of [amanmaqsood/prose-humanizer](https://github.com/amanmaqsood/prose-humanizer) (MIT), a writing +skill rather than a detector. They entered the packs the way everything here does: screened against +the calibration corpus first, where all six scored zero. Twelve other candidates from the same source +did not enter, because they fire on writing published before generative models existed. diff --git a/src/SignsOfAI.Core/Calibration/published-calibration.json b/src/SignsOfAI.Core/Calibration/published-calibration.json index cec0938..73bdcd3 100644 --- a/src/SignsOfAI.Core/Calibration/published-calibration.json +++ b/src/SignsOfAI.Core/Calibration/published-calibration.json @@ -1 +1 @@ -{"corpusId":"signsofai-human-baseline","texts":90,"measuredOn":"2026-08-05","engine":"0.4.0","recommendedThreshold":25,"flaggedAtThreshold":0,"rateLow":0,"rateHigh":0.04093562618368095,"noisiestRules":[{"ruleId":"stat.burstiness","textShare":0.2777777777777778},{"ruleId":"rhet.in-terms-of","textShare":0.1},{"ruleId":"rhet.not-only-but","textShare":0.08888888888888889},{"ruleId":"rhet.in-order-to","textShare":0.07777777777777778},{"ruleId":"lex.furthermore","textShare":0.07777777777777778},{"ruleId":"lex.robust","textShare":0.07777777777777778},{"ruleId":"lex.just","textShare":0.07777777777777778},{"ruleId":"lex.simply","textShare":0.07777777777777778}],"languages":[{"language":"en","texts":65,"recommendedThreshold":null,"bestBound":0.05580153215404492},{"language":"es","texts":25,"recommendedThreshold":null,"bestBound":0.13319225276039096}]} +{"corpusId":"signsofai-human-baseline","texts":90,"measuredOn":"2026-08-24","engine":"0.4.0","recommendedThreshold":25,"flaggedAtThreshold":0,"rateLow":0,"rateHigh":0.04093562618368095,"noisiestRules":[{"ruleId":"stat.burstiness","textShare":0.2777777777777778},{"ruleId":"rhet.in-terms-of","textShare":0.1},{"ruleId":"rhet.not-only-but","textShare":0.08888888888888889},{"ruleId":"rhet.in-order-to","textShare":0.07777777777777778},{"ruleId":"lex.furthermore","textShare":0.07777777777777778},{"ruleId":"lex.robust","textShare":0.07777777777777778},{"ruleId":"lex.just","textShare":0.07777777777777778},{"ruleId":"lex.simply","textShare":0.07777777777777778}],"languages":[{"language":"en","texts":65,"recommendedThreshold":null,"bestBound":0.05580153215404492},{"language":"es","texts":25,"recommendedThreshold":null,"bestBound":0.13319225276039096}]} diff --git a/src/SignsOfAI.Core/Rules/Packs/rules.en.json b/src/SignsOfAI.Core/Rules/Packs/rules.en.json index e480482..ce3e349 100644 --- a/src/SignsOfAI.Core/Rules/Packs/rules.en.json +++ b/src/SignsOfAI.Core/Rules/Packs/rules.en.json @@ -1114,6 +1114,66 @@ "severity": "Info", "message": "Mid-sentence bold — formatting slop.", "suggestion": "Emphasis rarely helps prose; cut the bold." + }, + { + "id": "chat.model-self-reference", + "category": "Rhetorical", + "regex": "\\bas an? (AI|artificial intelligence)( language)?( model| assistant)?\\b|\\bas a large language model\\b|\\bI(’|')?m an AI\\b", + "weight": 6.0, + "severity": "High", + "message": "A sentence the assistant says about itself, not a sentence in a document.", + "suggestion": "Delete it. If the text was drafted with a tool, say so in the acknowledgements instead.", + "evidence": "0 occurrences in the 249,455-word en half of the human calibration corpus (pre-2022, 65 texts)" + }, + { + "id": "chat.training-cutoff", + "category": "Rhetorical", + "regex": "\\bas of my (last )?(training|knowledge)[- ](update|cutoff|cut-off|data)\\b|\\bmy training data (only )?(goes|extends|runs) up to\\b|\\bas of my last update\\b", + "weight": 6.0, + "severity": "High", + "message": "A disclaimer about a model’s training data, left in the text.", + "suggestion": "Delete it, and check the claim it was hedging.", + "evidence": "0 occurrences in the 249,455-word en half of the human calibration corpus (pre-2022, 65 texts)" + }, + { + "id": "chat.signoff", + "category": "Rhetorical", + "regex": "\\bI hope th(is|at) helps\\b|\\bhope th(is|at) helps\\b|\\bwould you like me to\\b|\\blet me know if you(’|')?(d| would) like\\b|\\bis there anything else (I can|you(’|')?d like)\\b", + "weight": 6.0, + "severity": "High", + "message": "The closing line of an assistant’s reply, inside the document.", + "suggestion": "Delete it. The document has no reader to offer further help to.", + "evidence": "0 occurrences in the 249,455-word en half of the human calibration corpus (pre-2022, 65 texts)" + }, + { + "id": "chat.capability-disclaimer", + "category": "Rhetorical", + "regex": "\\bI (can(’|')?t|cannot|do not|don(’|')?t) (browse|access) (the )?(internet|web|real-?time)\\b|\\bI (do not|don(’|')?t) have (access to )?real-?time (data|information|access)\\b|\\bI(’|')?m unable to browse\\b", + "weight": 5.0, + "severity": "High", + "message": "A statement about what the writing tool cannot do.", + "suggestion": "Delete it, and supply the source it was standing in for.", + "evidence": "0 occurrences in the 249,455-word en half of the human calibration corpus (pre-2022, 65 texts)" + }, + { + "id": "chat.answer-preamble", + "category": "Rhetorical", + "regex": "\\bhere(’|')?s (the|a|your) (rewritten|revised|edited|updated|improved|polished) (version|text|draft|paragraph|essay|article)\\b|\\bhere is (the|a|your) (rewritten|revised|edited|updated|improved|polished) (version|text|draft|paragraph|essay|article)\\b", + "weight": 5.0, + "severity": "High", + "message": "The line a tool prints before handing back its answer.", + "suggestion": "Delete it. The document should start with the document.", + "evidence": "0 occurrences in the 249,455-word en half of the human calibration corpus (pre-2022, 65 texts)" + }, + { + "id": "chat.eager-opener", + "category": "Rhetorical", + "regex": "(?m)(?:^|(?<=[.!?])\\s)(Certainly|Of course|Absolutely|Sure thing|Great question)[!,]", + "weight": 3.5, + "severity": "Medium", + "message": "An assistant’s eager opener (“Certainly!”, “Great question!”).", + "suggestion": "Delete it. Nobody asked the document a question.", + "evidence": "0 occurrences in the 249,455-word en half of the human calibration corpus (pre-2022, 65 texts)" } ], "functionWords": [ diff --git a/src/SignsOfAI.Core/Rules/Packs/rules.es.json b/src/SignsOfAI.Core/Rules/Packs/rules.es.json index 1fde181..5b8aba3 100644 --- a/src/SignsOfAI.Core/Rules/Packs/rules.es.json +++ b/src/SignsOfAI.Core/Rules/Packs/rules.es.json @@ -916,6 +916,66 @@ "severity": "Info", "message": "Negrita a media frase — ruido de formato.", "suggestion": "El énfasis rara vez ayuda a la prosa; quita la negrita." + }, + { + "id": "chat.model-self-reference", + "category": "Rhetorical", + "regex": "\\bcomo (una? )?(modelo de lenguaje|IA|inteligencia artificial|asistente de (IA|inteligencia artificial))\\b|\\bsoy una (IA|inteligencia artificial)\\b", + "weight": 6.0, + "severity": "High", + "message": "Una frase que el asistente dice sobre sí mismo, no una frase del documento.", + "suggestion": "Bórrala. Si el texto se redactó con una herramienta, dilo en los agradecimientos.", + "evidence": "0 occurrences in the 39,712-word es half of the human calibration corpus (pre-2022, 25 texts)" + }, + { + "id": "chat.training-cutoff", + "category": "Rhetorical", + "regex": "\\bhasta mi última actualización\\b|\\bmis datos de entrenamiento\\b|\\bhasta donde (llega|alcanza) mi entrenamiento\\b|\\bmi conocimiento (llega|alcanza) hasta\\b", + "weight": 6.0, + "severity": "High", + "message": "Una advertencia sobre los datos de entrenamiento de un modelo, olvidada en el texto.", + "suggestion": "Bórrala, y comprueba la afirmación que estaba matizando.", + "evidence": "0 occurrences in the 39,712-word es half of the human calibration corpus (pre-2022, 25 texts)" + }, + { + "id": "chat.signoff", + "category": "Rhetorical", + "regex": "\\bespero que (esto |te |le )?(te |le )?(ayude|sirva|haya servido)\\b|¿(quieres|deseas|te gustaría) que (te )?(lo |la |los |las )?(reescriba|amplíe|resuma|ayude|prepare)\\b|\\bavísame si (quieres|necesitas|te)\\b|¿(hay )?algo más en lo que pueda ayudar", + "weight": 6.0, + "severity": "High", + "message": "La línea de cierre de una respuesta de asistente, dentro del documento.", + "suggestion": "Bórrala. El documento no tiene a quién ofrecerle más ayuda.", + "evidence": "0 occurrences in the 39,712-word es half of the human calibration corpus (pre-2022, 25 texts)" + }, + { + "id": "chat.capability-disclaimer", + "category": "Rhetorical", + "regex": "\\bno (tengo|puedo) (acceso a |navegar por |consultar )?(internet|la web|información en tiempo real)\\b|\\bno tengo acceso en tiempo real\\b", + "weight": 5.0, + "severity": "High", + "message": "Una declaración sobre lo que la herramienta de escritura no puede hacer.", + "suggestion": "Bórrala, y pon la fuente que estaba sustituyendo.", + "evidence": "0 occurrences in the 39,712-word es half of the human calibration corpus (pre-2022, 25 texts)" + }, + { + "id": "chat.answer-preamble", + "category": "Rhetorical", + "regex": "\\baquí (tienes|está|te dejo) (el|la|una|tu) (versión|texto|párrafo|redacción|artículo) (reescrit[ao]|revisad[ao]|corregid[ao]|mejorad[ao]|actualizad[ao])\\b|\\baquí (tienes|está) (el|la) (versión|texto) (reescrit[ao]|revisad[ao])\\b", + "weight": 5.0, + "severity": "High", + "message": "La línea que imprime una herramienta antes de entregar su respuesta.", + "suggestion": "Bórrala. El documento debería empezar por el documento.", + "evidence": "0 occurrences in the 39,712-word es half of the human calibration corpus (pre-2022, 25 texts)" + }, + { + "id": "chat.eager-opener", + "category": "Rhetorical", + "regex": "(?m)(?:^|(?<=[.!?¡])\\s)¡?(Por supuesto|Claro|Excelente pregunta|Desde luego)[!,]", + "weight": 3.5, + "severity": "Medium", + "message": "Apertura entusiasta de asistente («¡Por supuesto!», «¡Excelente pregunta!»).", + "suggestion": "Bórrala. Nadie le ha hecho una pregunta al documento.", + "evidence": "0 occurrences in the 39,712-word es half of the human calibration corpus (pre-2022, 25 texts)" } ], "functionWords": [ diff --git a/tests/SignsOfAI.Core.Tests/ChatResidueTests.cs b/tests/SignsOfAI.Core.Tests/ChatResidueTests.cs new file mode 100644 index 0000000..713e931 --- /dev/null +++ b/tests/SignsOfAI.Core.Tests/ChatResidueTests.cs @@ -0,0 +1,93 @@ +using System.Linq; +using SignsOfAI.Core; +using Xunit; + +namespace SignsOfAI.Core.Tests; + +/// +/// The half of the conversation that was never meant to be in the document. +/// +/// Every other rule in this project is a judgement about style, and a formal writer can lose to any +/// of them. These are not that. "I hope this helps" at the foot of an essay is not a register the +/// student chose; it is an assistant's closing line, pasted in with the answer — the same kind of +/// claim the character scanner makes, about where the file has been rather than about who is +/// talented. +/// +/// That is also why this batch was admissible when most of the pattern set it came from was not. +/// The rules were screened against the calibration corpus first — 249,455 words of English and +/// 39,712 of Spanish, all published before generative models existed — and every one of them scored +/// zero. Re-running the calibration afterwards left the published false-positive rate untouched, +/// which is the point: they cost nothing to carry. +/// +/// The mine they came from is amanmaqsood/prose-humanizer (MIT). Twelve of its other candidates +/// were rejected here by the same screen: underpin, optimize, elucidate, paradigm, exemplify and +/// illuminate are ordinary research English, and each appears in three to six of those ninety texts. +/// +public class ChatResidueTests +{ + private readonly AiWritingAnalyzer _a = new(); + + private bool Has(string text, string lang, string ruleId) => + _a.Analyze(text, lang).Findings.Any(f => f.RuleId == ruleId); + + [Theory] + [InlineData("As an AI language model, I should note that the figures are indicative.", "chat.model-self-reference")] + [InlineData("As of my last training update, the tidal survey had not been repeated.", "chat.training-cutoff")] + [InlineData("The wall was built in 1971. I hope this helps.", "chat.signoff")] + [InlineData("Would you like me to expand the section on sediment transport?", "chat.signoff")] + [InlineData("I cannot browse the internet, so the citation below is from memory.", "chat.capability-disclaimer")] + [InlineData("Here is the revised version of your essay on coastal erosion.", "chat.answer-preamble")] + [InlineData("Certainly! The coastline retreated by nine metres.", "chat.eager-opener")] + public void Flags_the_assistants_own_turn_en(string text, string ruleId) => + Assert.True(Has(text, "en", ruleId), ruleId); + + [Theory] + [InlineData("Como modelo de lenguaje, debo señalar que las cifras son indicativas.", "chat.model-self-reference")] + [InlineData("Hasta mi última actualización, el estudio no se había repetido.", "chat.training-cutoff")] + [InlineData("El muro se construyó en 1971. Espero que esto te ayude.", "chat.signoff")] + [InlineData("¿Quieres que amplíe el apartado sobre el transporte de sedimentos?", "chat.signoff")] + [InlineData("No tengo acceso a información en tiempo real sobre las mareas.", "chat.capability-disclaimer")] + [InlineData("Aquí tienes la versión reescrita de tu ensayo sobre la erosión costera.", "chat.answer-preamble")] + [InlineData("¡Por supuesto! La costa retrocedió nueve metros.", "chat.eager-opener")] + public void Flags_the_assistants_own_turn_es(string text, string ruleId) => + Assert.True(Has(text, "es", ruleId), ruleId); + + /// + /// The regexes have to leave ordinary writing alone, and two of them are close to sentences a + /// person really writes. A tutor's own feedback says "let me know" and a historian writes "of + /// course" mid-sentence; neither is an assistant handing back an answer. + /// + [Theory] + [InlineData("Let me know when the survey is finished and I will read it.", "chat.signoff")] + [InlineData("The wall was, of course, built long before the survey began.", "chat.eager-opener")] + [InlineData("I hope the council publishes the survey before the winter.", "chat.signoff")] + [InlineData("Here is the revised timetable the committee agreed on Tuesday.", "chat.answer-preamble")] + public void Leaves_a_person_writing_to_a_person_alone_en(string text, string ruleId) => + Assert.False(Has(text, "en", ruleId), ruleId); + + [Theory] + [InlineData("Avísame cuando termine el estudio y lo leo.", "chat.signoff")] + [InlineData("El muro, por supuesto, se construyó mucho antes del estudio.", "chat.eager-opener")] + [InlineData("Aquí tienes el calendario que acordó la comisión el martes.", "chat.answer-preamble")] + public void Leaves_a_person_writing_to_a_person_alone_es(string text, string ruleId) => + Assert.False(Has(text, "es", ruleId), ruleId); + + /// + /// A rule that fires on nothing measured is a rule with no measured human rate, and the pack + /// must keep saying so rather than inventing one. See PatternRule.HumanRatePer1000: an + /// absent rate means "never observed", which is what these are, and a rate of zero would be a + /// different and much stronger claim. + /// + [Theory] + [InlineData("en")] + [InlineData("es")] + public void The_new_rules_claim_no_human_rate(string lang) + { + var pack = SignsOfAI.Core.Rules.RulePackLoader.Load(lang); + var chat = pack.Patterns.Where(p => p.Id.StartsWith("chat.")).ToList(); + + Assert.Equal(6, chat.Count); + Assert.All(chat, rule => Assert.Null(rule.HumanRatePer1000)); + Assert.All(chat, rule => Assert.False(string.IsNullOrWhiteSpace(rule.Evidence), rule.Id)); + } +}