New docx template plus tweaks

backend/doc/config.py

@@ -175,7 +175,7 @@ class Settings(BaseSettings):
     UNORDERED_LIST_END_STR: str = "</ul>"
     VERSE_SPAN_FMT_STR: str = '<span class="verse">{}</span>'
     BOOK_NAME_FMT_STR: str = "<h2 class='book-name'>{}</h2>"
-    RESOURCE_TYPE_NAME_FMT_STR: str = "<h2 class='book-name'>{}</h2>"
+    RESOURCE_TYPE_NAME_FMT_STR: str = "<h1 class='book-name'>{}</h1>"
     HR: str = "<hr/>"
     TW_WORD_LIST_VERTICAL: bool = False
 

backend/doc/domain/assembly_strategies/assemble_by_book.py

@@ -27,7 +27,6 @@
 from doc.domain.parsing import handle_split_chapter_into_verses
 from doc.reviewers_guide.model import RGBook
 
-
 logger = settings.logger(__name__)
 
 

backend/doc/domain/assembly_strategies/assembly_strategy_utils.py

@@ -26,7 +26,6 @@
 from docx.oxml import parse_xml, OxmlElement
 from docx.oxml.ns import qn
 
-
 logger = settings.logger(__name__)
 
 
@@ -125,7 +124,7 @@ def tn_chapter_intro(
         and chapter_num in tn_book.chapters
         and tn_book.chapters[chapter_num].intro_html
     ):
-        content.append(tn_book.chapters[chapter_num].intro_html)
+        content.append(demote_headings_by_one(tn_book.chapters[chapter_num].intro_html))
     return "".join(content)
 
 
@@ -516,7 +515,6 @@ def get_chapter_intros(
     is_rtl: bool,
     show_tn_chapter_intro: bool,
     use_section_visual_separator: bool,
-    resource_type_name_fmt_str: str = settings.RESOURCE_TYPE_NAME_FMT_STR,
 ) -> list[DocumentPart]:
     document_parts: list[DocumentPart] = []
     if show_tn_chapter_intro:
@@ -891,9 +889,9 @@ def get_non_usfm_resources_chapter(
             use_section_visual_separator,
         )
     )
-    document_parts.extend(
-        rg_verses_parts(rg_book, chapter_num, is_rtl, use_section_visual_separator)
-    )
+    # document_parts.extend(
+    #     rg_verses_parts(rg_book, chapter_num, is_rtl, use_section_visual_separator)
+    # )
     return document_parts
 
 
@@ -978,7 +976,9 @@ def tnc_chapter_intro(
         and chapter_num in tnc_book.chapters
         and tnc_book.chapters[chapter_num].intro_html
     ):
-        content.append(tnc_book.chapters[chapter_num].intro_html)
+        content.append(
+            demote_headings_by_one(tnc_book.chapters[chapter_num].intro_html)
+        )
         if use_section_visual_separator:
             content.append(hr)
     return "".join(content)

backend/doc/domain/document_generator.py

@@ -758,22 +758,26 @@ def compose_docx_document(
     performing preprocessing and optional separators.
     """
     doc = Document()
-    html_to_docx = HtmlToDocx()
+    parser = HtmlToDocx(
+        tag_style_overrides={
+            "h1": "Title",
+        }
+    )
     t0 = time.time()
     for part in document_parts:
         if part.contained_in_two_column_section:
             add_two_column_section(doc)
         else:
             add_one_column_section(doc)
         try:
-            html_to_docx.add_html_to_document(part.content, doc)
+            parser.add_html_to_document(part.content, doc)
         except ValueError as e:
             logger.exception("Error converting HTML to docx: %s", e)
         if part.use_section_visual_separator:
             add_full_width_hr(doc)
         if part.add_page_break:
             add_page_break(doc)
-    style_superscripts(doc, lift_half_points=2, color=RGBColor(0x99, 0x99, 0x99))
+    style_superscripts(doc, lift_half_points=2, color=None)
     t1 = time.time()
     logger.info("Time for converting HTML to Docx: %.2f seconds", t1 - t0)
     return doc

backend/doc/utils/docx_util.py

@@ -167,15 +167,15 @@ def style_superscripts(
     doc: DocxDocument,
     *,
     lift_half_points: int = 2,
-    color: RGBColor = RGBColor(0x66, 0x66, 0x66),
+    color: RGBColor | None = None,
 ) -> None:
     """
     lift_half_points:
         2 = +1pt
         4 = +2pt
         6 = +3pt
     color:
-        RGBColor for superscripts (e.g. light gray)
+        RGBColor for superscripts (e.g. light gray) or None
     """
     for para in doc.paragraphs:
         runs = para.runs

backend/doc/utils/text_utils.py

@@ -2,7 +2,6 @@
 
 from doc.config import settings
 
-
 logger = settings.logger(__name__)
 
 HEADING_RE = compile(r"</?h([1-6])\b", IGNORECASE)
@@ -130,13 +129,6 @@ def demote_headings_by_one(content: str) -> str:
     )
 
 
-def demote_headings_by_two(content: str) -> str:
-    return HEADING_RE.sub(
-        lambda m: _demote_heading(m, levels=2),
-        content,
-    )
-
-
 if __name__ == "__main__":
 
     # To run the doctests in this module, in the root of the project do:

backend/requirements.in

@@ -1,9 +1,3 @@
-# TODO do we still need cython here? I think pyproject.toml build
-# dependencies brings it in for pydantic so it probably isn't needed
-# here. You have to test pydantic install to see though as it will
-# only use Rust backend if cython is detected
-# cython # For pydantic: https://pydantic-docs.helpmanual.io/install/
-# TODO do we still need aiofiles?
 aiofiles
 beautifulsoup4
 cachetools

frontend/tests/e2e/doc_test_2.ts

@@ -52,7 +52,7 @@ test('can select gateway tab after first selecting heart language and hitting ne
   await page.getByText('Cebuano').click()
   await page.getByRole('button', { name: 'Next' }).click()
   await page.getByRole('button', { name: 'Next' }).click()
-  await page.getByText('Regular').click({ timeout: 32_000 })
+  await page.getByText('Regular').click({ timeout: 120_000 })
   await page.getByText('Unlocked Literal Bible').click()
   await page.getByRole('button', { name: 'Next' }).click()
   await page.getByRole('radio', { name: 'PDF' }).click()

frontend/tests/e2e/doc_test_3.ts

@@ -40,7 +40,7 @@ test.skip('limit tw words switch', async ({ page }) => {
 
 test('use section visual separator setting', async ({ page }) => {
   await page.goto('http://localhost:8001/')
-  await page.getByText('Español Latin America (Latin').click()
+  await page.getByText('Español Latin America (Latin').click({ timeout: 120000 })
   await page.getByRole('button', { name: 'Next' }).click()
   await page.getByText('Efesios').click()
   await page.getByRole('button', { name: 'Next' }).click()

frontend/tests/e2e/passages_test.ts

@@ -1,6 +1,6 @@
 import { test, expect } from '@playwright/test'
 
-test.only('add passages', async ({ page }) => {
+test.skip('add passages', async ({ page }) => {
   await page.goto('http://localhost:8001/passages')
   await page.getByText('Español Latin America (Latin').click()
   await page.getByRole('button', { name: 'Next' }).click()

frontend/tests/e2e/passages_test_2.ts

@@ -30,7 +30,7 @@ test('checkboxes are only shown when language chosen has books in each checkbox
   // await expect(page.locator('body')).not.toContainText('Add OT Survey RG3 Passages')
   // await expect(page.locator('body')).not.toContainText('Add OT Survey RG4 Passages')
   await expect(page.locator('body')).toContainText("Add NT Survey Reviewers' Guide (RG) Passages", {
-    timeout: 32_000
+    timeout: 120_000
   })
   // await expect(page.locator('#stet-passages')).toContainText('Add STET Passages')
   await page.getByText("Add NT Survey Reviewers'").click()
@@ -44,7 +44,7 @@ test('checkboxes are only shown when language chosen has books in each checkbox
   await expect(page.locator('body')).toContainText(
     "Add all OT Survey Reviewers' Guide (RG) Passages",
     {
-      timeout: 32_000
+      timeout: 120_000
     }
   )
   await expect(page.locator('#add-ot-rg1-passages')).toContainText(