diff --git a/src/routes/blog/postgresql-support-in-torrust-tracker/+page.server.ts b/src/routes/blog/postgresql-support-in-torrust-tracker/+page.server.ts new file mode 100644 index 0000000..5cf0731 --- /dev/null +++ b/src/routes/blog/postgresql-support-in-torrust-tracker/+page.server.ts @@ -0,0 +1,14 @@ +import { getMetadata } from '$lib/data/metadata'; +import type { PageServerLoad } from './$types'; + +export const load: PageServerLoad = async ({ url }) => { + const slug = url.pathname.split('/').filter(Boolean).pop(); + if (!slug) throw new Error('Slug could not be determined.'); + + const metadata = await getMetadata(); + const currentPost = metadata.find((post) => post.slug === slug); + + if (!currentPost) throw new Error(`Post not found: ${slug}`); + + return { currentPost, allPosts: metadata }; +}; diff --git a/src/routes/blog/postgresql-support-in-torrust-tracker/+page.svelte b/src/routes/blog/postgresql-support-in-torrust-tracker/+page.svelte new file mode 100644 index 0000000..ca135f7 --- /dev/null +++ b/src/routes/blog/postgresql-support-in-torrust-tracker/+page.svelte @@ -0,0 +1,819 @@ + + + + +
+ +
+

Introduction

+

+ We are excited to announce that + Torrust Tracker + now supports + PostgreSQL + as a first-class database backend, alongside the existing + SQLite and + MySQL drivers. + This has been a long-standing feature request, and it is the result of a long journey that involved + a full overhaul of the persistence layer, new tooling, and valuable contributions from the community. +

+

+ In this post we will walk through the history of this feature, explain what changed under + the hood, highlight the new tooling built along the way, and share our plans for the + upcoming major release. +

+ +

A Long-Requested Feature

+

+ The first request for PostgreSQL support was filed back on September 20, 2023 + in + issue #462. PostgreSQL is widely used in production environments and it was a natural fit for + operators who already run PostgreSQL infrastructure and want to avoid introducing a second + database engine. +

+

+ At the time, however, adding PostgreSQL support was not straightforward. The tracker's + persistence layer was synchronous, using the + r2d2 + connection pool crate with + rusqlite + and + mysql. The problem is that the + postgres + crate wraps + tokio_postgres, which tries to spawn a nested + Tokio runtime — + something that conflicts with the tracker's existing async runtime. The community + PR #1684 worked around this by spawning a fresh OS thread per database operation, but that approach + was rejected as a performance concern: under load, each query would pay the cost of OS thread + creation and context switching. +

+

+ The real fix was to migrate all drivers to an async-native library. Rather than bolt + PostgreSQL on top of the synchronous stack with a workaround, we decided to do the right + thing first: replace r2d2 / rusqlite / mysql with + async + sqlx across all drivers, and then add PostgreSQL on top of that clean foundation. +

+ +

The Persistence Overhaul EPIC

+

+ In May 2025 we opened + EPIC #1525 — Overhaul persistence, which became the umbrella for all the work needed before PostgreSQL could be added + cleanly. The driving insight was the + Martin Fowler quote we kept returning to: +

+
"Make the change easy, then make the easy change."
+

+ The EPIC was broken down into two phases and nine sequential sub-issues, each + independently reviewable and mergeable into develop. +

+ +

Phase 1 — Make the Change Easy

+

Before touching a single PostgreSQL file, we had to modernise the persistence stack:

+
    +
  1. + Persistence test coverage — A compatibility-matrix CI workflow that + tests the tracker against a range of supported SQLite, MySQL, and PostgreSQL versions + (see the + DB Compatibility Matrix section below). +
  2. +
  3. + qBittorrent end-to-end runner — A Rust binary that runs a full seeder → tracker + → leecher download using real, containerised qBittorrent clients, so we can verify correct + behaviour at the protocol level. +
  4. +
  5. + Persistence benchmarking — A benchmark runner (persistence_benchmark_runner) that measures per-operation latency for each driver using --driver, + --db-version, and --ops flags, and outputs a JSON report for easy + diffing across runs. +
  6. +
  7. + Split persistence traits — The monolithic Database trait + was split into four narrow context traits (SchemaMigrator, + TorrentMetricsStore, WhitelistStore, + AuthKeyStore) plus a blanket aggregate supertrait, reducing coupling and + making each driver easier to implement and test in isolation. +
  8. +
  9. + Migrate SQLite and MySQL to sqlx — Both existing drivers were rewritten + using async sqlx connection pools, replacing the old synchronous + r2d2 / rusqlite / mysql stack. +
  10. +
  11. + Introduce schema migrations — Raw DDL was replaced with + sqlx::migrate!(), and a legacy-bootstrap path was added to history-align + databases that were created before migrations existed. +
  12. +
  13. + Align Rust and DB types — The MySQL download-counter columns were + widened from INTEGER (signed 32-bit, max ~2.1 billion) to + BIGINT via a versioned migration. The Rust type + NumberOfDownloads stays u32 — the wider column is intentional; the + application type bounds writes at compile time. +
  14. +
+ +

Phase 2 — Make the Easy Change

+

+ With the foundation in place, adding PostgreSQL (sub-issue 1525-08, + tracked in + issue #1723) became straightforward: +

+
    +
  • + A new Driver::PostgreSQL variant in the configuration crate (serialises as + "postgresql"). +
  • +
  • + Four PostgreSQL migration files, timestamp-aligned with the existing SQLite/MySQL + history. +
  • +
  • + A new postgres.rs driver implementing all four narrow traits via async + sqlx. +
  • +
  • Wiring in the driver factory and setup dispatch.
  • +
  • + Testcontainers-based driver tests and environment-gated execution. +
  • +
  • + The compatibility-matrix, qBittorrent E2E runner, and benchmark runner all extended for + PostgreSQL. +
  • +
  • Default PostgreSQL container config and updated documentation.
  • +
+ +

A Community-Driven Feature

+

+ A key part of this story is the contribution from community member + DamnCrab. They opened + PR #1684 + with an initial PostgreSQL implementation using + r2d2_postgres, which started the conversation about the right approach. Their follow-up + PR #1695 + incorporated review feedback, and reviewer guidance was provided in + PR #1700 + to help bridge the gap. +

+

Three ideas from DamnCrab's work were retained in the final implementation:

+
    +
  • + The DB compatibility matrix script to validate tracker compatibility across database + versions. +
  • +
  • + End-to-end tests using a real BitTorrent client (containerised + qBittorrent). +
  • +
  • + Basic database benchmarking to compare persistence performance before/after the sqlx migration and across database engines. +
  • +
+

+ Thank you, DamnCrab, for the contributions and ideas that made their way into the final + result. +

+ +

New Tooling

+

+ The overhaul produced several tools that are useful beyond just adding PostgreSQL support: +

+

DB Compatibility Matrix

+

+ A + GitHub Actions workflow (db-compatibility.yaml) + that runs the tracker's full driver test suite against a matrix of database versions on every + push and pull request: +

+
    +
  • MySQL: 8.0, 8.4
  • +
  • PostgreSQL: 14, 15, 16, 17
  • +
+

+ Each combination spins up a real container via + testcontainers + and runs the driver tests with the db-compatibility-tests feature flag. This explicitly + documents which database versions the tracker is compatible with, and protects users who run + infrastructure with a version that differs from what the developers typically test against. + If a new database release introduces a breaking change, the matrix catches it before it reaches + users. +

+ +

qBittorrent End-to-End Runner

+

+ A + Rust + binary that runs a complete BitTorrent transfer — seeder announces, leecher connects, download + completes — using real + qBittorrent clients running in containers. This is the closest we can get to a real-world integration + test without deploying to production. +

+

Benchmark Runner

+

+ The + persistence_benchmark_runner + is a developer binary that measures the persistence-layer operations implemented by the + Database trait. It benchmarks one driver per invocation and prints a JSON + report to standard output with per-operation timing statistics: count, + best, + median, and worst in microseconds. +

+

Run it with:

+ +

+ The output is plain JSON so you can redirect it to a file, diff runs, or feed it into any + visualisation tool: +

+ .benchmarks/bench-results-sqlite3.json`} + /> +

A sample report looks like this:

+ +

+ It is intentionally simple — there is no built-in comparison mode. The value comes from + running it before and after a change (or across drivers) and diffing the JSON files. This + makes it easy to catch performance regressions early, without the overhead of a full + criterion benchmark suite. +

+

+ There is also a + GitHub Actions workflow (db-benchmarking.yaml) + that runs the benchmark against all three drivers on every push and pull request. It runs with + --ops 10 — just enough to confirm the binary builds and executes cleanly — rather + than producing statistically significant numbers. The main purpose in CI is to catch compilation + breakages and driver-level errors early, not to track performance over time. +

+ + + Detailed documentation for all three tools is available in the + Torrust Tracker repository. + + +

Benchmark Results

+

+ With all three drivers in place we ran the benchmark runner for the first time across all + engines on the same machine (AMD Ryzen 9 7950X, Ubuntu 25.10, Docker 28.3.3) with + --ops 100. The full report is available in the repository at + docs/benchmarking/runs/2026-05-01/REPORT.md. +

+ +

Total benchmark time

+ + + + + + + + + + + + + +
DriverTotal (ms)
SQLite3119 ms
MySQL 8.46 372 ms
MySQL 8.07 272 ms
PostgreSQL 171 451 ms
+ +

Per-operation medians (µs)

+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
OperationSQLite3MySQL 8.4MySQL 8.0PostgreSQL 17
save_torrent_downloads89769984298
load_torrent_downloads2311211588
load_all_torrents_downloads77172171146
increase_downloads_for_torrent707731 005302
save_global_downloads767931 066299
load_global_downloads2111513786
increase_global_downloads677741 036305
add_info_hash_to_whitelist81735981294
get_info_hash_from_whitelist2110911895
load_whitelist55161175135
remove_info_hash_from_whitelist81766962293
add_key_to_keys81750974292
get_key_from_keys2211812995
load_keys77167189155
remove_key_from_keys73739994300
+ +

Takeaways

+
    +
  • + SQLite3 is the fastest for single-node, embedded use cases — no network round-trip, + everything in-process. It is best suited for development, testing, or single-server deployments + where concurrent write load is low. +
  • +
  • + PostgreSQL 17 comfortably beats both MySQL versions for write operations. + Write medians (~290–305 µs) are roughly 2.5–3× faster than MySQL 8.0 and ~60% faster than + MySQL 8.4. +
  • +
  • + Read performance is comparable between PostgreSQL 17 and MySQL 8.4 for + simple lookups; aggregate reads (load_*) are slightly slower on PostgreSQL. +
  • +
  • + Overall PostgreSQL is significantly faster than MySQL in total benchmark + time (1 451 ms vs 6 372 ms for MySQL 8.4), driven primarily by faster write operations. +
  • +
+

+ These numbers are the first PostgreSQL baseline. Future runs will track regressions as the + persistence layer continues to evolve, and will provide a growing picture of how the + drivers compare over time. +

+ +

What's Next

+

+ PostgreSQL support is already merged into the develop branch and will be + included in the next major release (v4.0.0), which we are planning to + ship this year — though no date is set yet. If you want to try it out today, you can build + from + develop and set the driver in your tracker configuration: +

+ +

+ A default Docker Compose configuration for PostgreSQL is also included in the repository, + so you can spin up a local instance with a single command. +

+

Resources & External Links

+ +
+
+
+ + +
+ + diff --git a/src/routes/blog/postgresql-support-in-torrust-tracker/metadata.ts b/src/routes/blog/postgresql-support-in-torrust-tracker/metadata.ts new file mode 100644 index 0000000..362a7cc --- /dev/null +++ b/src/routes/blog/postgresql-support-in-torrust-tracker/metadata.ts @@ -0,0 +1,12 @@ +export const metadata = { + title: 'PostgreSQL Support in Torrust Tracker', + slug: 'postgresql-support-in-torrust-tracker', + contributor: 'Jose Celano', + contributorSlug: 'jose-celano', + date: '2026-05-01T12:00:00.000Z', + coverImage: + '/images/posts/postgresql-support-in-torrust-tracker/postgresql-support-in-torrust-tracker.webp', + excerpt: + 'Torrust Tracker now supports PostgreSQL as a first-class database backend. Learn about the journey from a community feature request to a full persistence overhaul, the new tools built along the way, and what this means for the upcoming major release.', + tags: ['Rust', 'PostgreSQL', 'Announcement', 'Release'] +}; diff --git a/static/blogMetadata.json b/static/blogMetadata.json index 901668b..6b4fb82 100644 --- a/static/blogMetadata.json +++ b/static/blogMetadata.json @@ -13,19 +13,17 @@ ] }, { - "title": "Building with AI Agents, Building for AI Agents", - "slug": "building-with-ai-agents-building-for-ai-agents", + "title": "Containerizing Rust Applications", + "slug": "containerizing-rust-applications-best-practices", "contributor": "Jose Celano", "contributorSlug": "jose-celano", - "date": "2026-02-27T12:00:00.000Z", - "coverImage": "/images/posts/building-with-ai-agents-building-for-ai-agents/ai-agent-developer-tools-workflow.webp", - "excerpt": "I spent months building the Torrust Tracker Deployer without writing a single line of code directly — every line came from a GitHub Copilot agent. But this post is about something bigger: the shift in thinking that came from that experience and what it taught us about designing tools that AI agents can actually use well.", + "date": "2023-11-09T12:08:04.295Z", + "coverImage": "/images/posts/rust-crab-carrying-a-shipping-container.jpeg", + "excerpt": "Torrust services (Tracker and Index) support docker, we want to ensure that contributors understand our containerfile and we also want to share good practices for containerizing Rust applications.", "tags": [ - "AI", - "Rust", - "Torrust", - "DevOps", - "Tutorial" + "Torrent", + "Tracker", + "BitTorrent" ] }, { @@ -72,17 +70,19 @@ ] }, { - "title": "Containerizing Rust Applications", - "slug": "containerizing-rust-applications-best-practices", + "title": "Building with AI Agents, Building for AI Agents", + "slug": "building-with-ai-agents-building-for-ai-agents", "contributor": "Jose Celano", "contributorSlug": "jose-celano", - "date": "2023-11-09T12:08:04.295Z", - "coverImage": "/images/posts/rust-crab-carrying-a-shipping-container.jpeg", - "excerpt": "Torrust services (Tracker and Index) support docker, we want to ensure that contributors understand our containerfile and we also want to share good practices for containerizing Rust applications.", + "date": "2026-02-27T12:00:00.000Z", + "coverImage": "/images/posts/building-with-ai-agents-building-for-ai-agents/ai-agent-developer-tools-workflow.webp", + "excerpt": "I spent months building the Torrust Tracker Deployer without writing a single line of code directly — every line came from a GitHub Copilot agent. But this post is about something bigger: the shift in thinking that came from that experience and what it taught us about designing tools that AI agents can actually use well.", "tags": [ - "Torrent", - "Tracker", - "BitTorrent" + "AI", + "Rust", + "Torrust", + "DevOps", + "Tutorial" ] }, { @@ -181,20 +181,6 @@ "BitTorrent" ] }, - { - "title": "Announcing the Beta v3.0.0 Live Demo", - "slug": "live-demo-beta-v3", - "contributor": "Jose Celano", - "contributorSlug": "jose-celano", - "date": "2023-12-15T12:08:04.295Z", - "coverImage": "/images/posts/rust-crab-happy.jpg", - "excerpt": "We will release a new major version v3.0.0. We want the community to test it before the final release while it's still in Beta. You can contribute to make Torrust better.", - "tags": [ - "Announcement", - "Demo", - "Documentation" - ] - }, { "title": "Introducing the Torrust Tracker Deployer", "slug": "introducing-the-torrust-tracker-deployer", @@ -209,6 +195,20 @@ "Announcement" ] }, + { + "title": "Announcing the Beta v3.0.0 Live Demo", + "slug": "live-demo-beta-v3", + "contributor": "Jose Celano", + "contributorSlug": "jose-celano", + "date": "2023-12-15T12:08:04.295Z", + "coverImage": "/images/posts/rust-crab-happy.jpg", + "excerpt": "We will release a new major version v3.0.0. We want the community to test it before the final release while it's still in Beta. You can contribute to make Torrust better.", + "tags": [ + "Announcement", + "Demo", + "Documentation" + ] + }, { "title": "The New Torrust Tracker Demo Is Live", "slug": "new-torrust-tracker-demo", @@ -225,23 +225,6 @@ "Torrust Tracker" ] }, - { - "title": "Profiling the Torrust BitTorrent UDP Tracker", - "slug": "profiling-the-torrust-bittorrent-udp-tracker", - "contributor": "Jose Celano", - "contributorSlug": "jose-celano", - "date": "2024-03-25T00:00:00.000Z", - "coverImage": "/images/posts/profiling-the-torrust-bittorrent-udp-tracker/profiling-the-torrust-bittorrent-udp-tracker.webp", - "excerpt": "Dive into the advanced profiling techniques driving the Torrust BitTorrent Tracker's performance. This post uncovers the tools and methods we leverage to ensure the tracker's efficiency, scalability, and our journey towards optimizing peer-to-peer file sharing.", - "tags": [ - "Performance", - "Profiling", - "UDP Tracker", - "Flamegraph", - "Kcachegrind", - "Valgrind" - ] - }, { "title": "How nf_conntrack Overflow Causes Intermittent UDP Tracker Downtime with Docker", "slug": "nf-conntrack-overflow-docker-udp-tracker", @@ -273,6 +256,38 @@ "Deployment" ] }, + { + "title": "Profiling the Torrust BitTorrent UDP Tracker", + "slug": "profiling-the-torrust-bittorrent-udp-tracker", + "contributor": "Jose Celano", + "contributorSlug": "jose-celano", + "date": "2024-03-25T00:00:00.000Z", + "coverImage": "/images/posts/profiling-the-torrust-bittorrent-udp-tracker/profiling-the-torrust-bittorrent-udp-tracker.webp", + "excerpt": "Dive into the advanced profiling techniques driving the Torrust BitTorrent Tracker's performance. This post uncovers the tools and methods we leverage to ensure the tracker's efficiency, scalability, and our journey towards optimizing peer-to-peer file sharing.", + "tags": [ + "Performance", + "Profiling", + "UDP Tracker", + "Flamegraph", + "Kcachegrind", + "Valgrind" + ] + }, + { + "title": "PostgreSQL Support in Torrust Tracker", + "slug": "postgresql-support-in-torrust-tracker", + "contributor": "Jose Celano", + "contributorSlug": "jose-celano", + "date": "2026-05-01T12:00:00.000Z", + "coverImage": "/images/posts/postgresql-support-in-torrust-tracker/postgresql-support-in-torrust-tracker.webp", + "excerpt": "Torrust Tracker now supports PostgreSQL as a first-class database backend. Learn about the journey from a community feature request to a full persistence overhaul, the new tools built along the way, and what this means for the upcoming major release.", + "tags": [ + "Rust", + "PostgreSQL", + "Announcement", + "Release" + ] + }, { "title": "Released version v3.0.0-beta", "slug": "released-v3-0-0", @@ -330,22 +345,6 @@ "Deployment" ] }, - { - "title": "Submitting Trackers to newTrackon", - "slug": "submitting-trackers-to-newtrackon", - "contributor": "Jose Celano", - "contributorSlug": "jose-celano", - "date": "2026-04-01T12:00:00.000Z", - "coverImage": "/images/posts/submitting-trackers-to-newtrackon/submitting-bittorrent-tracker-to-newtrackon.webp", - "excerpt": "A practical guide to submitting BitTorrent trackers to newTrackon, the popular third-party uptime monitoring service. Learn the prerequisites, how to work around the one-tracker-per-IP constraint using floating IPs, and how to configure DNS correctly with BEP34.", - "tags": [ - "BitTorrent", - "Tracker", - "DevOps", - "DNS", - "Demo" - ] - }, { "title": "Setting Up Torrust with Claude Code", "slug": "setting-up-torrust-with-claude-code", @@ -374,6 +373,22 @@ "BitTorrent" ] }, + { + "title": "Submitting Trackers to newTrackon", + "slug": "submitting-trackers-to-newtrackon", + "contributor": "Jose Celano", + "contributorSlug": "jose-celano", + "date": "2026-04-01T12:00:00.000Z", + "coverImage": "/images/posts/submitting-trackers-to-newtrackon/submitting-bittorrent-tracker-to-newtrackon.webp", + "excerpt": "A practical guide to submitting BitTorrent trackers to newTrackon, the popular third-party uptime monitoring service. Learn the prerequisites, how to work around the one-tracker-per-IP constraint using floating IPs, and how to configure DNS correctly with BEP34.", + "tags": [ + "BitTorrent", + "Tracker", + "DevOps", + "DNS", + "Demo" + ] + }, { "title": "Torrust - Enhancing the BitTorrent Ecosystem", "slug": "torrust-enhancing-the-bittorrent-ecosystem", @@ -418,20 +433,6 @@ "Community" ] }, - { - "title": "What is a BitTorrent tracker and types of trackers", - "slug": "what-is-a-bittorent-tracker", - "contributor": "", - "contributorSlug": "", - "date": "2023-10-05T13:42:25.671Z", - "coverImage": "/images/posts/tracker.jpg", - "excerpt": "Basic explanation of what a BitTorrent tracker is and the two types of trackers, public and private.", - "tags": [ - "Torrent", - "Tracker", - "BitTorrent" - ] - }, { "title": "Vortex: A High-Performance Rust BitTorrent Client for Linux", "slug": "vortex-rust-bittorrent-client-review", @@ -447,5 +448,19 @@ "Performance", "Third-party" ] + }, + { + "title": "What is a BitTorrent tracker and types of trackers", + "slug": "what-is-a-bittorent-tracker", + "contributor": "", + "contributorSlug": "", + "date": "2023-10-05T13:42:25.671Z", + "coverImage": "/images/posts/tracker.jpg", + "excerpt": "Basic explanation of what a BitTorrent tracker is and the two types of trackers, public and private.", + "tags": [ + "Torrent", + "Tracker", + "BitTorrent" + ] } ] \ No newline at end of file diff --git a/static/images/posts/postgresql-support-in-torrust-tracker/postgresql-support-in-torrust-tracker.webp b/static/images/posts/postgresql-support-in-torrust-tracker/postgresql-support-in-torrust-tracker.webp new file mode 100644 index 0000000..4638d3f Binary files /dev/null and b/static/images/posts/postgresql-support-in-torrust-tracker/postgresql-support-in-torrust-tracker.webp differ