diff --git a/AGENTS.md b/AGENTS.md index 208afe24..8ef1e48f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -32,3 +32,7 @@ Read [CONTRIBUTING.md](CONTRIBUTING.md) for code style, linting (e.g. `make lint - Base the commit message on the diff of the file that you are going to commit - Prefer one line commit messages - Do not co-author commits + +## Subdirectory guidance + +- [docs/AGENTS.md](docs/AGENTS.md) - documentation review and reference page focus diff --git a/docs/.pages b/docs/.pages index 1b153383..a77446c5 100644 --- a/docs/.pages +++ b/docs/.pages @@ -5,3 +5,5 @@ nav: - conformance.md - Reference: - reference + - Project: + - project diff --git a/docs/AGENTS.md b/docs/AGENTS.md new file mode 100644 index 00000000..e1739366 --- /dev/null +++ b/docs/AGENTS.md @@ -0,0 +1,17 @@ +# Documentation guidance + +- Review documentation diffs with the same rigor as code diffs; apply the + project documentation conventions whenever docs files are touched. +- Keep reference pages centered on the API or object named by the page; do not + add examples for unrelated loaders or implementation escape hatches. +- Put the primary API autodoc or mkdocstrings block near the top of reference + pages, after any prerequisite admonition and before explanatory prose; do not + add a separate `API Reference` heading before it. +- On the first prose mention of an external software product on a docs page, + link to the official product or package page and include the relevant MkDocs + Material icon; do not force links inside code spans, Python object names, + filenames, options, or generated example output. +- When a docs page is influenced by an ADR, add a bottom-of-page collapsed + admonition named `Decisions` that links to the relevant decision document. +- This file is excluded from MkDocs publishing through `exclude_docs` in + `mkdocs.yml`. diff --git a/docs/conformance.md b/docs/conformance.md index b30e1f96..03522889 100644 --- a/docs/conformance.md +++ b/docs/conformance.md @@ -13,6 +13,8 @@ PyLD aims to conform with: - [Universal RDF Graph Canonicalization Algorithm 2012 (URGNA2012)](https://www.w3.org/TR/rdf-canon/#URDNA2012) - The JSON-LD Working Group [test suite](https://github.com/w3c/json-ld-api/tree/master/tests) +PyLD aims to follow [JSON-LD Best Practices](https://w3c.github.io/json-ld-bp/). + The test runner is updated over time to note or skip newer tests that are not yet supported. diff --git a/docs/examples/document_loaders/sqlite_cache_basic.py b/docs/examples/document_loaders/sqlite_cache_basic.py new file mode 100644 index 00000000..08a3e7e6 --- /dev/null +++ b/docs/examples/document_loaders/sqlite_cache_basic.py @@ -0,0 +1,19 @@ +import json +from pathlib import Path + +from pyld import SqliteCacheRequestsDocumentLoader, jsonld + +doc = { + "@context": {"name": "http://schema.org/name"}, + "name": "Earth", +} + +loader = SqliteCacheRequestsDocumentLoader( + sqlite_file_path=Path("/tmp/pyld_example_http_cache.sqlite"), +) + +# At the first execution, this will perform a network request to https://schema.org, +# but subsequent executions of the same code will not, using a cached response. +result = jsonld.expand(doc, options={"documentLoader": loader}) + +print(json.dumps(result, indent=2)) diff --git a/docs/installation.md b/docs/installation.md index 7659708f..1e0d2a48 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -13,6 +13,7 @@ matching extra: ```bash pip install "PyLD[requests]" pip install "PyLD[aiohttp]" +pip install "PyLD[requests-cache]" ``` You can also depend on `requests` or `aiohttp` directly if your project already diff --git a/docs/javascripts/mermaid.mjs b/docs/javascripts/mermaid.mjs new file mode 100644 index 00000000..de171579 --- /dev/null +++ b/docs/javascripts/mermaid.mjs @@ -0,0 +1,8 @@ +import mermaid from 'https://cdn.jsdelivr.net/npm/mermaid@11/dist/mermaid.esm.min.mjs'; + +mermaid.initialize({ + startOnLoad: false, + securityLevel: 'loose', +}); + +window.mermaid = mermaid; diff --git a/docs/project/decisions/use-requests-cache-for-sync-http-caching-in-document-loaders.md b/docs/project/decisions/use-requests-cache-for-sync-http-caching-in-document-loaders.md new file mode 100644 index 00000000..40f54e48 --- /dev/null +++ b/docs/project/decisions/use-requests-cache-for-sync-http-caching-in-document-loaders.md @@ -0,0 +1,183 @@ +--- +title: Use requests-cache for persistent HTTP caching in synchronous Python code +status: decided +date: 2026-06-29 +author: Anatoly Scherbakov +tags: [decision] +hide: [toc] +--- + +# Use `requests-cache` for persistent HTTP caching in synchronous Python code + +{{ adr_metadata(date, status) }} + +## :material-text-box-outline: Context + +[:material-file-download-outline: PyLD document loaders](/pyld/reference/document-loaders/) fetch remote JSON-LD contexts and other documents over HTTP. [JSON-LD Best Practices](https://w3c.github.io/json-ld-bp/) recommends: + +> **Best Practice 14:** Cache JSON-LD Contexts +> +> Services providing a JSON-LD Context *SHOULD* set HTTP cache-control headers to allow liberal caching of such contexts, and clients *SHOULD* attempt to use a locally cached version of these documents. +> +> — [§ 8.1 Cache JSON-LD Contexts](https://w3c.github.io/json-ld-bp/#cache-json-ld-contexts) + +### Status Quo + +PyLD already caches remote contexts in process memory only. [`ContextResolver`](https://github.com/digitalbazaar/pyld/blob/master/lib/pyld/context_resolver.py) shares a module-level [`LRUCache`](https://github.com/digitalbazaar/pyld/blob/master/lib/pyld/jsonld.py#L160-L162) of resolved contexts, and [`ResolvedContext`](https://github.com/digitalbazaar/pyld/blob/master/lib/pyld/resolved_context.py) keeps a per-document [`LRUCache`](https://github.com/digitalbazaar/pyld/blob/master/lib/pyld/resolved_context.py#L28) for processed contexts relative to an active context. + +[`FrozenDocumentLoader`](/pyld/reference/document-loaders/frozen/) serves files from disk based on its configuration options set in code. + +None of this is HTTP-aware or persistent across processes: [`RequestsDocumentLoader`](/pyld/reference/document-loaders/requests/) and [`AioHttpDocumentLoader`](/pyld/reference/document-loaders/aiohttp/) issue a fresh HTTP request on every load with no `Cache-Control` handling. + +### HTTP Caching Standards + +Client-side caching semantics are governed by the following standards: + +```mermaid +flowchart LR + rfc2616("RFC 2616
HTTP/1.1") + rfc7234("RFC 7234
HTTP/1.1: Caching") + rfc9111("RFC 9111
HTTP Caching") + rfc9110("RFC 9110
HTTP Semantics") + rfc5861("RFC 5861
Cache-Control Extensions for Stale Content") + rfc8246("RFC 8246
HTTP Immutable Responses") + + rfc2616 -->|is superseded by| rfc7234 + rfc7234 -->|is superseded by| rfc9111 + rfc9110 -->|complements| rfc9111 + rfc5861 -->|complements| rfc9111 + rfc8246 -->|complements| rfc9111 + + click rfc2616 "https://www.rfc-editor.org/rfc/rfc2616.html" "RFC 2616" + click rfc7234 "https://www.rfc-editor.org/rfc/rfc7234.html" "RFC 7234" + click rfc9111 "https://www.rfc-editor.org/rfc/rfc9111.html" "RFC 9111" + click rfc9110 "https://www.rfc-editor.org/rfc/rfc9110.html" "RFC 9110" + click rfc5861 "https://www.rfc-editor.org/rfc/rfc5861.html" "RFC 5861" + click rfc8246 "https://www.rfc-editor.org/rfc/rfc8246.html" "RFC 8246" +``` + +This ADR compares Python HTTP caching libraries that could extend document loaders with standards-aware, persistent caching. + +### Alternatives Rejected + +| Library | HTTP-aware? | Persistent? | +|---------|:----------:|:----------:| +| [:fontawesome-brands-github: `tkem/cachetools`](https://github.com/tkem/cachetools) | :x: | :x: | +| [:fontawesome-brands-github: `grantjenks/python-diskcache`](https://github.com/grantjenks/python-diskcache) | :x: | :white_check_mark: | + +Neither library implements HTTP cache semantics — `cachetools` is a generic in-memory structure and `diskcache` is persistent but not HTTP-aware — so they do not fit document-loader HTTP caching. + +## :material-arrow-decision-outline: Decision + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
[:fontawesome-brands-github: `requests-cache/requests-cache`](https://github.com/requests-cache/requests-cache)[:fontawesome-brands-github: `psf/cachecontrol`](https://github.com/psf/cachecontrol)[:fontawesome-brands-github: `requests-cache/aiohttp-client-cache`](https://github.com/requests-cache/aiohttp-client-cache)[:fontawesome-brands-github: `karpetrosyan/hishel`](https://github.com/karpetrosyan/hishel)
Backend
[:fontawesome-brands-github: `psf/requests`](https://github.com/psf/requests)[:white_check_mark:](https://requests-cache.readthedocs.io/en/stable/#quickstart)[:white_check_mark:](https://cachecontrol.readthedocs.io/en/latest/usage.html#wrapper)[:x:](https://aiohttp-client-cache.readthedocs.io/en/stable/#aiohttp-client-cache)[:white_check_mark:](https://hishel.com/requests.html)
[:fontawesome-brands-github: `encode/httpx`](https://github.com/encode/httpx)[:x:](https://requests-cache.readthedocs.io/en/stable/#summary)[:x:](https://cachecontrol.readthedocs.io/en/latest/#welcome-to-cachecontrol-s-documentation)[:x:](https://aiohttp-client-cache.readthedocs.io/en/stable/#aiohttp-client-cache)[:white_check_mark:](https://hishel.com/quickstart.html#with-httpx)
[:fontawesome-brands-github: `aio-libs/aiohttp`](https://github.com/aio-libs/aiohttp)[:x:](https://requests-cache.readthedocs.io/en/stable/project_info/related_projects.html#client-side-http-caching)[:x:](https://cachecontrol.readthedocs.io/en/latest/#welcome-to-cachecontrol-s-documentation)[:white_check_mark:](https://aiohttp-client-cache.readthedocs.io/en/stable/#quickstart)[:x:](https://github.com/karpetrosyan/hishel#-features)
HTTP Headers Support
`Cache-Control` request[Broad directive support.](https://requests-cache.readthedocs.io/en/stable/user_guide/headers.html#supported-headers)[`max-age`, `no-cache`, `no-store`, `min-fresh`; other directives parsed but not necessarily honored.](https://cachecontrol.readthedocs.io/en/latest/usage.html)[Simplified: `max-age`, `no-cache`, `no-store`.](https://aiohttp-client-cache.readthedocs.io/en/stable/user_guide.html#cache-control)[:white_check_mark:](https://www.rfc-editor.org/rfc/rfc9111.html#name-request-directives)
`Cache-Control` response[Broad directive support.](https://requests-cache.readthedocs.io/en/stable/user_guide/headers.html#supported-headers)[`max-age`, `no-store`, validator-driven caching.](https://cachecontrol.readthedocs.io/en/latest/etags.html)[Simplified: `max-age`, `no-store`.](https://aiohttp-client-cache.readthedocs.io/en/stable/user_guide.html#cache-control)[:white_check_mark:](https://www.rfc-editor.org/rfc/rfc9111.html#name-response-directives)
`Expires`[:white_check_mark:](https://requests-cache.readthedocs.io/en/stable/user_guide/headers.html#supported-headers)[:white_check_mark:](https://cachecontrol.readthedocs.io/en/latest/etags.html)[:white_check_mark:](https://aiohttp-client-cache.readthedocs.io/en/stable/user_guide.html#cache-control)[:white_check_mark:](https://www.rfc-editor.org/rfc/rfc9111.html#name-expires)
`ETag` / `If-None-Match`[:white_check_mark:](https://requests-cache.readthedocs.io/en/stable/user_guide/headers.html#supported-headers)[:white_check_mark:](https://cachecontrol.readthedocs.io/en/latest/etags.html)[:warning:](https://aiohttp-client-cache.readthedocs.io/en/stable/modules/aiohttp_client_cache.cache_control.html#aiohttp_client_cache.cache_control.compose_refresh_headers)[:white_check_mark:](https://www.rfc-editor.org/rfc/rfc9111.html#name-validation)
`Last-Modified` / `If-Modified-Since`[:white_check_mark:](https://requests-cache.readthedocs.io/en/stable/user_guide/headers.html#supported-headers)[:white_check_mark:](https://cachecontrol.readthedocs.io/en/latest/etags.html)[:warning:](https://aiohttp-client-cache.readthedocs.io/en/stable/modules/aiohttp_client_cache.cache_control.html#aiohttp_client_cache.cache_control.compose_refresh_headers)[:white_check_mark:](https://www.rfc-editor.org/rfc/rfc9111.html#name-validation)
`Vary`[:white_check_mark:](https://requests-cache.readthedocs.io/en/stable/user_guide/headers.html#supported-headers)[:white_check_mark:](https://github.com/psf/cachecontrol/blob/master/cachecontrol/serialize.py#L56-L66)[:x:](https://aiohttp-client-cache.readthedocs.io/en/stable/user_guide.html#cache-control)[:white_check_mark:](https://www.rfc-editor.org/rfc/rfc9111.html#name-calculating-cache-keys-with)
Meta *(last updated: 28 June 2026)*
Last release[1.3.2](https://pypi.org/project/requests-cache/1.3.2/)
2026-05-11
[0.14.4](https://pypi.org/project/CacheControl/0.14.4/)
2025-11-14
[0.14.3](https://pypi.org/project/aiohttp-client-cache/0.14.3/)
2026-01-07
[1.3.0](https://pypi.org/project/hishel/1.3.0/)
2026-06-11
GitHub Stars:star: [1,495](https://github.com/requests-cache/requests-cache/stargazers):star: [499](https://github.com/psf/cachecontrol/stargazers):star: [152](https://github.com/requests-cache/aiohttp-client-cache/stargazers):star: [395](https://github.com/karpetrosyan/hishel/stargazers)
Decision:white_check_mark: Adopt as optional sync HTTP cache support for [`RequestsDocumentLoader`](/pyld/reference/document-loaders/requests/).:x: Exclude — shares the `requests` backend with `requests-cache`, but [defaults to an in-memory store](https://cachecontrol.readthedocs.io/en/latest/#quick-start) with no first-class persistent backend comparable to [`requests-cache` storage options](https://requests-cache.readthedocs.io/en/stable/user_guide/backends.html), offers narrower [`Cache-Control` directive support](https://cachecontrol.readthedocs.io/en/latest/usage.html), and trails on maintenance signals in this comparison.:question: Defer for async support to limit dependency and maintenance footprint; candidate if [`AioHttpDocumentLoader`](/pyld/reference/document-loaders/aiohttp/) gains HTTP caching.:question: Defer — strong HTTP cache policy design, but adoption would target [`httpx`](https://github.com/karpetrosyan/hishel#-features) rather than PyLD's current [`AioHttpDocumentLoader`](/pyld/reference/document-loaders/aiohttp/).
+ +:warning: marks indirect or limited support. `aiohttp-client-cache` has [conditional refresh support for validators](https://aiohttp-client-cache.readthedocs.io/en/stable/modules/aiohttp_client_cache.cache_control.html#aiohttp_client_cache.cache_control.compose_refresh_headers), but its own documentation describes cache-header handling as a [simplified subset](https://aiohttp-client-cache.readthedocs.io/en/stable/user_guide.html#cache-control) rather than full automatic validator-driven HTTP cache revalidation. For `hishel`, header rows link to the governing [RFC 9111](https://www.rfc-editor.org/rfc/rfc9111.html) sections because its default [`SpecificationPolicy`](https://hishel.com/policies) implements the full specification rather than documenting per-header support separately. + +## :material-arrow-right-bold-outline: Consequences + +- PyLD will add optional, HTTP-aware sync caching via `requests-cache` without making caching a required dependency. +- Opt-in sync caching will now be provided by [`SqliteCacheRequestsDocumentLoader`](/pyld/reference/document-loaders/sqlite-cache-requests/), composing `RequestsDocumentLoader` with a persistent SQLite `CachedSession`. +- A custom in-process HTTP cache implementation is rejected; with a fitting library available, PyLD will integrate `requests-cache` rather than expand the codebase to implement RFC 9111 caching itself. +- CacheControl is excluded for the sync `requests` path in favor of `requests-cache`. +- Async HTTP caching remains out of scope for this decision; `aiohttp-client-cache` and `hishel` stay as future candidates. diff --git a/docs/project/index.md b/docs/project/index.md new file mode 100644 index 00000000..e42be8e9 --- /dev/null +++ b/docs/project/index.md @@ -0,0 +1,37 @@ +--- +hide: [toc] +--- + +# :material-hard-hat: Project + +Project documentation for PyLD — architecture decisions, development notes, and other material that sits alongside the API reference. + +## :material-gavel: Architecture Decision Records + +Architecture Decision Records (ADRs) document the technical choices taken during development of PyLD. + +
+ +!!! success inline "[Use `requests-cache` for persistent HTTP caching in synchronous Python code](decisions/use-requests-cache-for-sync-http-caching-in-document-loaders/)" + :material-calendar-clock: 29 June 2026 + + Choose `requests-cache` for persistent, HTTP-aware synchronous + document-loader caching. + +!!! quote inline "Document the next important project decision" + When a technical choice changes PyLD's architecture, public API, dependencies, + or long-term maintenance path, capture the reasoning here as an + [Architecture Decision Record](https://adr.github.io/). + +
+ +## :material-source-branch: Decision Process + +ADRs are useful when a technical choice has meaningful consequences for PyLD's +architecture, public API, dependencies, or long-term maintenance. A PR +discussion is enough to establish consensus for an ADR: if the maintainers agree +with the decision and merge the PR, the ADR lands as `decided`. + +When consensus has not been reached yet, an ADR can land as `draft` and be +refined in follow-up PRs. If a later decision replaces an earlier one, mark the +earlier ADR as `superseded` and link to the new decision. diff --git a/docs/reference/document-loaders/index.md b/docs/reference/document-loaders/index.md index c784d2bc..8e6607a9 100644 --- a/docs/reference/document-loaders/index.md +++ b/docs/reference/document-loaders/index.md @@ -12,6 +12,13 @@ class-based loaders for common cases and supports custom subclasses of Synchronous remote document loading with `requests`. +- [:material-database:{ .lg .middle } `SqliteCacheRequestsDocumentLoader`](sqlite-cache-requests.md) + + --- + + Persistent [:simple-sqlite: SQLite](https://www.sqlite.org/) caching for + remote JSON-LD documents. + - [:material-sync:{ .lg .middle } `AioHttpDocumentLoader`](aiohttp.md) --- diff --git a/docs/reference/document-loaders/sqlite-cache-requests.md b/docs/reference/document-loaders/sqlite-cache-requests.md new file mode 100644 index 00000000..05024d61 --- /dev/null +++ b/docs/reference/document-loaders/sqlite-cache-requests.md @@ -0,0 +1,51 @@ +--- +hide: [toc] +--- + +# :material-database: `SqliteCacheRequestsDocumentLoader` + +!!! info "Prerequisite" + This functionality requires an extra dependency, [:simple-pypi: `requests-cache`](https://pypi.org/project/requests-cache). Install `PyLD` with: + + ```shell + pip install 'PyLD[requests-cache]' + ``` + +::: pyld.SqliteCacheRequestsDocumentLoader + options: + show_root_heading: false + show_bases: false + heading_level: 3 + +When `sqlite_file_path` is omitted, the cache defaults to the platform user +cache directory: + +| OS | Default path | +|----|--------------| +| :simple-linux: Linux | `~/.cache/pyld/http_cache.sqlite` | +| :simple-apple: macOS | `~/Library/Caches/pyld/http_cache.sqlite` | +| :material-microsoft-windows-classic: Windows | `%LOCALAPPDATA%\pyld\http_cache.sqlite` | + +`SqliteCacheRequestsDocumentLoader` retrieves remote JSON-LD documents and keeps +them in a persistent [:simple-sqlite: SQLite](https://www.sqlite.org/) cache. It +is useful for applications that repeatedly load the same remote contexts across +process runs. + +HTTP cache headers (`Cache-Control`, `Expires`, validators) are honored, so +publishers can control how cached context documents are reused. + +[JSON-LD Best Practices](https://w3c.github.io/json-ld-bp/) recommends: + +> **Best Practice 14:** Cache JSON-LD Contexts +> +> Services providing a JSON-LD Context *SHOULD* set HTTP cache-control headers to +> allow liberal caching of such contexts, and clients *SHOULD* attempt to use a +> locally cached version of these documents. +> +> - [§ 8.1 Cache JSON-LD Contexts](https://w3c.github.io/json-ld-bp/#cache-json-ld-contexts) + +{{ example('document_loaders/sqlite_cache_basic.py', output_syntax='json') }} + +??? note "Decisions" + This page is influenced by the decision to + [use `requests-cache` for persistent HTTP caching in synchronous Python code](/pyld/project/decisions/use-requests-cache-for-sync-http-caching-in-document-loaders/). diff --git a/docs/requirements.txt b/docs/requirements.txt index 3d1f8802..92b44a43 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -4,3 +4,4 @@ mkdocs-material==9.7.6 mkdocs-macros-plugin==1.5.0 mkdocs-awesome-pages-plugin==2.10.1 mkdocstrings[python]>=0.30 +requests-cache>=1.3 diff --git a/docs/stylesheets/extra.css b/docs/stylesheets/extra.css index 097493ff..bb1f3935 100644 --- a/docs/stylesheets/extra.css +++ b/docs/stylesheets/extra.css @@ -64,3 +64,27 @@ margin-left: auto; font-weight: normal; } + +.md-typeset #adr-index { + display: flow-root; +} + +/* ADR comparison column tints — match mkdocs-material admonition hues at 10%. */ +:root, +[data-md-color-scheme="slate"] { + --adr-col-included-bg: color-mix(in srgb, #00c853 10%, transparent); + --adr-col-undecided-bg: color-mix(in srgb, #ff9100 10%, transparent); + --adr-col-rejected-bg: color-mix(in srgb, #ff1744 10%, transparent); +} + +.md-typeset :is(th, td).adr-col-included { + background-color: var(--adr-col-included-bg); +} + +.md-typeset :is(th, td).adr-col-undecided { + background-color: var(--adr-col-undecided-bg); +} + +.md-typeset :is(th, td).adr-col-rejected { + background-color: var(--adr-col-rejected-bg); +} diff --git a/docs_macros.py b/docs_macros.py index b49ccae6..1a3c3643 100644 --- a/docs_macros.py +++ b/docs_macros.py @@ -3,8 +3,13 @@ import re import subprocess import sys +from datetime import date from pathlib import Path +import yaml +from mkdocs.utils.meta import YAML_RE +from yaml import SafeLoader + ROOT_DIR = Path(__file__).resolve().parent EXAMPLES_DIR = ROOT_DIR / 'docs' / 'examples' sys.path.insert(0, str(ROOT_DIR / 'lib')) @@ -130,7 +135,95 @@ def _example_github_url(name, repo_url): return f'{repo_url.rstrip("/")}/blob/{branch}/{rel_path.as_posix()}' +def _human_date(value): + if not value: + return '' + parsed = date.fromisoformat(value) if isinstance(value, str) else value + return f'{parsed.day} {parsed.strftime("%B %Y")}' + + +_ADR_STATUS = { + 'draft': (':material-pencil-outline:', 'Draft'), + 'undecided': (':question:', 'Undecided'), + 'decided': (':white_check_mark:', 'Decided'), +} + + +_ADR_STATUS_ADMONITION = { + 'draft': 'note', + 'undecided': 'warning', + 'decided': 'success', +} + + +def _adr_status_label(value): + if not value: + return '' + key = str(value).lower() + return _ADR_STATUS.get(key, (None, str(value).replace('_', ' ').title()))[1] + + +def _adr_metadata_date(value): + if not value: + return '' + return f':material-calendar-clock: {_human_date(value)}' + + +def _adr_metadata(date, status): + kind = _ADR_STATUS_ADMONITION.get(str(status).lower(), 'note') + parts = [] + label = _adr_status_label(status) + if label: + parts.append(label) + date_part = _adr_metadata_date(date) + if date_part: + parts.append(date_part) + title = ' · '.join(parts) + return f'!!! {kind} "{title}"\n' + + +def _adr_status(value): + if not value: + return '' + key = str(value).lower() + icon, label = _ADR_STATUS.get( + key, + (':material-information-outline:', str(value).replace('_', ' ').title()), + ) + return f'{icon} {label}' + + +def _adr_status_icon(value): + if not value: + return ':material-information-outline:' + key = str(value).lower() + return _ADR_STATUS.get( + key, + (':material-information-outline:', ''), + )[0] + + +def _parse_frontmatter(path): + text = path.read_text(encoding='utf-8-sig') + match = YAML_RE.match(text) + if not match: + return {} + return yaml.load(match.group(1), SafeLoader) or {} + + def define_env(env): + @env.filter + def human_date(value): + return _human_date(value) + + @env.filter + def adr_status(value): + return _adr_status(value) + + @env.macro + def adr_metadata(date, status): + return _adr_metadata(date, status) + @env.macro def bundled_contexts_table(): from pyld import BUNDLED_CONTEXTS diff --git a/lib/pyld/__init__.py b/lib/pyld/__init__.py index 42de12b5..ae05cb95 100644 --- a/lib/pyld/__init__.py +++ b/lib/pyld/__init__.py @@ -6,6 +6,7 @@ from .documentloader.base import DocumentLoader, RemoteDocument from .documentloader.frozen import BUNDLED_CONTEXTS, FrozenDocumentLoader from .documentloader.requests import RequestsDocumentLoader +from .documentloader.requests_sqlite_cache import SqliteCacheRequestsDocumentLoader __all__ = [ 'AioHttpDocumentLoader', @@ -15,5 +16,6 @@ 'FrozenDocumentLoader', 'RequestsDocumentLoader', 'RemoteDocument', + 'SqliteCacheRequestsDocumentLoader', 'jsonld', ] diff --git a/lib/pyld/documentloader/requests.py b/lib/pyld/documentloader/requests.py index 9ef9caf6..4a6e6e3b 100644 --- a/lib/pyld/documentloader/requests.py +++ b/lib/pyld/documentloader/requests.py @@ -15,16 +15,18 @@ from pyld import iri_resolver from pyld.documentloader.base import DocumentLoader, RemoteDocument -from pyld.jsonld import JsonLdError, parse_link_header, LINK_HEADER_REL +from pyld.jsonld import LINK_HEADER_REL, JsonLdError, parse_link_header class RequestsDocumentLoader(DocumentLoader): """Remote document loader using Requests.""" - def __init__(self, secure=False, **kwargs): - import requests + def __init__(self, secure=False, session=None, **kwargs): + if session is None: + import requests - self.requests = requests + session = requests.Session() + self.session = session self.secure = secure self.kwargs = kwargs @@ -61,7 +63,7 @@ def __call__(self, url, options=None) -> RemoteDocument: headers = { 'Accept': 'application/ld+json, application/json' } - response = self.requests.get(url, headers=headers, **self.kwargs) + response = self.session.get(url, headers=headers, **self.kwargs) content_type = response.headers.get('content-type') if not content_type: diff --git a/lib/pyld/documentloader/requests_sqlite_cache.py b/lib/pyld/documentloader/requests_sqlite_cache.py new file mode 100644 index 00000000..d99dfbcf --- /dev/null +++ b/lib/pyld/documentloader/requests_sqlite_cache.py @@ -0,0 +1,55 @@ +""" +SQLite-backed HTTP cache document loader using requests-cache. + +.. module:: jsonld.documentloader.requests_sqlite_cache + :synopsis: Persistent SQLite HTTP caching for Requests document loader +""" +from pathlib import Path + +from pyld.documentloader.base import DocumentLoader, RemoteDocument +from pyld.documentloader.requests import RequestsDocumentLoader + + +def _resolve_sqlite_file_path(sqlite_file_path: Path | None) -> Path: + """Return absolute path to the SQLite cache file.""" + from platformdirs import user_cache_dir + + if sqlite_file_path is None: + return (Path(user_cache_dir('pyld')) / 'http_cache.sqlite').resolve() + path = Path(sqlite_file_path).expanduser() + if not path.is_absolute(): + raise ValueError( + 'sqlite_file_path must be an absolute path to the .sqlite file') + return path.resolve() + + +class SqliteCacheRequestsDocumentLoader(DocumentLoader): + """Remote document loader with persistent SQLite HTTP caching. + + :param secure: require all requests to use HTTPS (default: False). + :param sqlite_file_path: absolute path to the ``.sqlite`` cache file; when + omitted, defaults to the platform user cache directory under ``pyld/``. + """ + + def __init__( + self, + secure: bool = False, + *, + sqlite_file_path: Path | None = None, + ): + from requests_cache import CachedSession + + path = _resolve_sqlite_file_path(sqlite_file_path) + self.session = CachedSession( + cache_name=str(path), + backend='sqlite', + cache_control=True, + # Cache JSON-LD contexts persistently by default; Cache-Control and + # related response headers still override this when present. + expire_after=-1, + ) + self._loader = RequestsDocumentLoader( + secure=secure, session=self.session) + + def __call__(self, url, options=None) -> RemoteDocument: + return self._loader(url, options=options) diff --git a/mkdocs.yml b/mkdocs.yml index 4fad1cb3..4d2a59db 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -3,10 +3,15 @@ site_description: Python implementation of the JSON-LD API site_url: https://digitalbazaar.github.io/pyld/ repo_url: https://github.com/digitalbazaar/pyld repo_name: digitalbazaar/pyld +exclude_docs: | + **/AGENTS.md extra_css: - stylesheets/extra.css +extra_javascript: + - javascripts/mermaid.mjs + theme: name: material features: @@ -29,12 +34,18 @@ markdown_extensions: emoji_generator: !!python/name:material.extensions.emoji.to_svg - pymdownx.highlight: anchor_linenums: true - - pymdownx.superfences + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format - pymdownx.tabbed: alternate_style: true slugify: !!python/object/apply:pymdownx.slugs.slugify kwds: case: lower + - pymdownx.tasklist: + custom_checkbox: true - toc: permalink: true diff --git a/requirements-test.txt b/requirements-test.txt index 28a66eae..8584b40a 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -2,3 +2,4 @@ ruff pytest pytest-cov typing_extensions +requests-cache>=1.3 diff --git a/setup.py b/setup.py index 62f533b1..0fc11db4 100644 --- a/setup.py +++ b/setup.py @@ -61,6 +61,7 @@ extras_require={ 'requests': ['requests'], 'aiohttp': ['aiohttp'], + 'requests-cache': ['requests-cache>=1.3'], 'cachetools': ['cachetools'], 'frozendict': ['frozendict'], } diff --git a/tests/test_sqlite_cache_requests_document_loader.py b/tests/test_sqlite_cache_requests_document_loader.py new file mode 100644 index 00000000..3006a8d7 --- /dev/null +++ b/tests/test_sqlite_cache_requests_document_loader.py @@ -0,0 +1,125 @@ +"""Tests for SqliteCacheRequestsDocumentLoader and HTTP cache behavior.""" + +import json +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path + +import pytest + +from pyld import ( + DocumentLoader, + RequestsDocumentLoader, + SqliteCacheRequestsDocumentLoader, +) +from pyld.documentloader.requests_sqlite_cache import _resolve_sqlite_file_path + +requests_cache = pytest.importorskip('requests_cache') +CachedSession = requests_cache.CachedSession + + +class _ContextHandler(BaseHTTPRequestHandler): + request_count = 0 + + def do_GET(self): + type(self).request_count += 1 + body = json.dumps({ + '@context': {'name': 'http://example.org/name'}, + }).encode() + self.send_response(200) + self.send_header('Content-Type', 'application/ld+json') + self.send_header('Cache-Control', 'max-age=3600') + self.end_headers() + self.wfile.write(body) + + def log_message(self, format, *args): + pass + + +@pytest.fixture +def context_url(): + """Local HTTP server returning JSON-LD with Cache-Control: max-age=3600.""" + _ContextHandler.request_count = 0 + server = HTTPServer(('127.0.0.1', 0), _ContextHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + url = f'http://127.0.0.1:{port}/context.jsonld' + yield url + server.shutdown() + + +def test_requests_document_loader_accepts_custom_session(): + """RequestsDocumentLoader accepts a CachedSession via session=.""" + loader = RequestsDocumentLoader( + session=CachedSession(backend='memory', cache_control=True)) + assert isinstance(loader, DocumentLoader) + assert callable(loader) + loader.session.close() + + +def test_sqlite_cache_requests_document_loader_is_document_loader(): + """Sqlite loader is a DocumentLoader composing RequestsDocumentLoader.""" + loader = SqliteCacheRequestsDocumentLoader() + assert isinstance(loader, DocumentLoader) + assert isinstance(loader._loader, RequestsDocumentLoader) + assert callable(loader) + loader.session.close() + + +def test_sqlite_cache_requests_document_loader_rejects_relative_sqlite_file_path(): + """Relative sqlite_file_path is rejected.""" + with pytest.raises(ValueError, match='absolute path'): + SqliteCacheRequestsDocumentLoader( + sqlite_file_path=Path('relative.sqlite')) + + +def test_sqlite_cache_file_path_is_resolved(tmp_path): + """Absolute sqlite_file_path is normalized to a full path.""" + sqlite_file_path = tmp_path / 'cache' / '..' / 'contexts.sqlite' + assert _resolve_sqlite_file_path(sqlite_file_path) == ( + tmp_path / 'contexts.sqlite').resolve() + + +def test_http_cache_headers_serve_from_cache_with_cache_control(context_url): + """With cache_control=True, Cache-Control max-age avoids a second HTTP hit.""" + loader = RequestsDocumentLoader( + session=CachedSession( + 'test_memory_cache_control', + backend='memory', + cache_control=True, + )) + loader(context_url) + loader(context_url) + assert _ContextHandler.request_count == 1 + loader.session.close() + + +def test_http_cache_headers_without_cache_control_hits_server_twice(context_url): + """With cache_control=False, response Cache-Control headers are ignored.""" + loader = RequestsDocumentLoader( + session=CachedSession( + 'test_memory_no_cache_control', + backend='memory', + cache_control=False, + expire_after=0, + )) + loader(context_url) + loader(context_url) + assert _ContextHandler.request_count == 2 + loader.session.close() + + +def test_sqlite_cache_requests_document_loader_persists(context_url, tmp_path): + """Second loader instance reuses the on-disk SQLite cache.""" + cache_path = tmp_path / 'contexts.sqlite' + loader = SqliteCacheRequestsDocumentLoader(sqlite_file_path=cache_path) + loader(context_url) + assert _ContextHandler.request_count == 1 + loader.session.close() + + _ContextHandler.request_count = 0 + loader = SqliteCacheRequestsDocumentLoader(sqlite_file_path=cache_path) + loader(context_url) + assert _ContextHandler.request_count == 0 + loader.session.close()