stream_http: add libcurl-based HTTP/HTTPS backend

.github/workflows/build.yml

@@ -310,7 +310,7 @@ jobs:
             brew update
           fi
           if brew list ffmpeg &> /dev/null; then brew unlink ffmpeg; fi
-          brew install -q autoconf automake pkgconf libtool python freetype fribidi little-cms2 \
+          brew install -q autoconf automake curl pkgconf libtool python freetype fribidi little-cms2 \
             luajit libass ffmpeg-full meson rust uchardet mujs libplacebo molten-vk vulkan-loader vulkan-headers \
             libarchive libbluray libcaca libcdio-paranoia libdvdnav rubberband zimg
           brew link ffmpeg-full
@@ -439,6 +439,7 @@ jobs:
           apk update
           apk add \
               binutils \
+              curl-dev \
               ffmpeg-dev \
               gcc \
               git \
@@ -493,6 +494,7 @@ jobs:
         run: |
             sudo pkg_add -U \
                 cmake \
+                curl \
                 ffmpeg \
                 git \
                 libarchive \
@@ -541,6 +543,7 @@ jobs:
             sudo pkg install -y \
                 alsa-lib \
                 cmake \
+                curl \
                 evdev-proto \
                 ffmpeg \
                 git \
@@ -625,6 +628,7 @@ jobs:
             ca-certificates:p
             cc:p
             cppwinrt:p
+            curl:p
             ffmpeg:p
             lcms2:p
             libarchive:p

DOCS/interface-changes/http-backend.txt

@@ -0,0 +1 @@
+add libcurl-based stream backend (http, https, ftp, ftps), with new options `--curl-enabled`, `--curl-http-version`, `--curl-max-redirects`, `--curl-max-retries`, `--curl-connect-timeout`, `--curl-buffer-size` and `--curl-max-request-size`

DOCS/man/options.rst

@@ -5718,6 +5718,75 @@ Network
     The bitrate as used is sent by the server, and there's no guarantee it's
     actually meaningful.
 
+Network backend (libcurl)
+-------------------------
+
+When mpv is built with libcurl support, ``http://``, ``https://``, ``ftp://``
+and ``ftps://`` URLs are served by an internal libcurl-based stream backend
+instead of FFmpeg. The backend fully supports all features of libcurl, making it
+more robust and compatible with a wide range of servers and CDNs, and faster too.
+
+For HTTP transfers, the backend transparently negotiates HTTP/1.1, HTTP/2
+multiplexing or HTTP/3 (QUIC) when the server offers them, with HSTS enabled
+and TCP keep-alive turned on. Content compression (gzip, deflate, zstd,
+brotli) is always advertised in the request. If the server applies it, the
+transfer is treated as non-seekable. Servers normally do not compress
+already-compressed media payloads. Otherwise, it's great improvement for text
+playlist data transfers.
+
+The backend honors the network options listed above (``--user-agent``,
+``--http-proxy``, ``--http-header-fields``, ``--referrer``, ``--cookies*``,
+``--tls-*``).
+
+If libcurl is not available at build time, mpv uses FFmpeg's networking
+implementation instead.
+
+To inspect libcurl's debug output (requests, response headers,
+TLS/connection diagnostics), set ``--msg-level=curl=trace``.
+
+``--curl-enabled=<yes|no>``
+    Enable the libcurl-based network backend (default: ``yes``).
+
+    Defaults to ``no`` with libavformat < 62.15.101, which has a nested IO
+    cleanup bug that can cause crashes or memory leaks. The issue happens only
+    on transfer failures or aborts, can be enabled if you don't mind possible
+    stability issues.
+
+``--curl-http-version=<auto|1.0|1.1|2|2tls|2-prior-knowledge|3|3only>``
+    Select the maximum HTTP protocol version libcurl is allowed to negotiate.
+    If libcurl was built without HTTP/3 support, it will fallback to ``auto``.
+    (default: ``auto``, i.e. let libcurl pick)
+
+``--curl-max-redirects=<0-100>``
+    Maximum number of HTTP redirects to follow before reporting an error
+    (default: 16).
+
+``--curl-max-retries=<0-100>``
+    Number of times a single seekable transfer may be transparently
+    re-attempted after a recoverable error (timeout, connection drop,
+    HTTP/2 stream reset, ...) before the stream gives up (default: 5).
+    Non-seekable transfers cannot be retried and ignore this option.
+
+``--curl-connect-timeout=<seconds>``
+    TCP/TLS connect timeout in seconds (default: 30, range 0-600). 0 lets
+    libcurl use its built-in default. The overall transfer timeout is
+    controlled by ``--network-timeout``.
+
+``--curl-buffer-size=<bytes>``
+    Size of the per-stream producer-side ring buffer that decouples the
+    network thread from the consumer (default: 4 MiB, minimum: 32 KiB).
+    Lower values may reduce in-flight data and reduce latency.
+
+``--curl-max-request-size=<bytes>``
+    For seekable streams, split the transfer into Range requests of at most
+    this size (default: 0, i.e. one open-ended request for the whole stream).
+    A non-zero value can help with very long-running connections that some
+    CDNs or proxies recycle aggressively, and is also a common workaround for
+    per-connection bandwidth throttling employed by some CDNs (notably some
+    video hosting services), where each individual Range request is served at
+    full speed but a single long-lived connection is rate-limited. Ignored for
+    non-seekable streams.
+
 DVB
 ---
 

ci/build-mingw64.sh

@@ -324,14 +324,25 @@ _subrandr () {
 }
 _subrandr_mark=lib/libsubrandr.dll.a
 
+_curl () {
+    local ver=8.20.0
+    gettar "https://curl.se/download/curl-${ver}.tar.xz"
+    builddir curl-${ver}
+    cmake .. "${cmake_args[@]}" \
+        -DCURL_{USE_SCHANNEL,ZLIB}=ON -DCURL_DISABLE_LDAP=ON -DCURL_USE_LIBPSL=OFF
+    makeplusinstall
+    popd
+}
+_curl_mark=lib/libcurl.dll.a
+
 for x in iconv zlib-ng shaderc spirv-cross amf-headers nv-headers dav1d lcms2; do
     build_if_missing $x
 done
 if [[ "$TARGET" != "i686-"* ]]; then
     build_if_missing vulkan-headers
     build_if_missing vulkan-loader
 fi
-for x in ffmpeg libplacebo freetype fribidi harfbuzz libass luajit; do
+for x in ffmpeg libplacebo freetype fribidi harfbuzz libass luajit curl; do
     build_if_missing $x
 done
 if [[ "$TARGET" != "i686-"* ]]; then
@@ -358,7 +369,7 @@ mpv_args=(
     -Dmujs:werror=false
     -Dmujs:default_library=static
     -Dlua=luajit
-    -D{amf,shaderc,spirv-cross,d3d11,javascript}=enabled
+    -D{amf,shaderc,spirv-cross,d3d11,javascript,libcurl}=enabled
 )
 meson setup $build "${mpv_args[@]}"
 meson compile -C $build
@@ -384,7 +395,7 @@ if [ "$2" = pack ]; then
         av*.dll sw*.dll postproc-[0-9]*.dll
         # everything else
         subrandr-[0-9]*.dll lib{ass,freetype,fribidi,harfbuzz,iconv,placebo}-[0-9]*.dll
-        lib{shaderc_shared,spirv-cross-c-shared,dav1d,lcms2,zlib1}.dll
+        lib{curl,shaderc_shared,spirv-cross-c-shared,dav1d,lcms2,zlib1}.dll
     )
     [[ -f vulkan-1.dll ]] && dlls+=(vulkan-1.dll)
     mv -v "${dlls[@]}" ..

ci/build-win32.ps1

@@ -322,6 +322,7 @@ meson setup build `
     -Dlibplacebo:tests=false `
     -Dlibplacebo:vulkan=enabled `
     -Dlibplacebo:d3d11=enabled `
+    -Dlibpsl:tests=false `
     -Dxxhash:inline-all=true `
     -Dxxhash:cli=false `
     -Dluajit:amalgam=true `

common/global.h

@@ -11,6 +11,7 @@ struct mpv_global {
     char *configdir;
     struct stats_base *stats;
     struct demux_packet_pool *packet_pool;
+    struct curl_ctx *curl;
 };
 
 #endif

demux/demux_lavf.c

@@ -49,6 +49,8 @@
 #include "misc/thread_tools.h"
 
 #include "stream/stream.h"
+#include "stream/stream_curl.h"
+
 #include "demux.h"
 #include "stheader.h"
 #include "options/m_config.h"
@@ -214,6 +216,7 @@ static const struct format_hack format_hacks[] = {
 struct nested_stream {
     AVIOContext *id;
     int64_t last_bytes;
+    void *curl_data;
 };
 
 struct stream_info {
@@ -945,12 +948,24 @@ static int nested_io_open(struct AVFormatContext *s, AVIOContext **pb,
         }
     }
 
-    int r = priv->default_io_open(s, pb, url, flags, options);
+    // Try the libcurl-based backend first, so nested connections use the same
+    // stack. Only ENOSYS (the URL not supported) falls back to Lavf IO.
+    void *curl_data = NULL;
+    int r = AVERROR(ENOSYS);
+#if HAVE_LIBCURL
+    r = mp_curl_avio_open(demuxer, pb, &curl_data, url, flags, options,
+                          s->protocol_whitelist, s->protocol_blacklist);
+    mp_assert(r != 0 || curl_data != NULL);
+#endif
+    if (r == AVERROR(ENOSYS))
+        r = priv->default_io_open(s, pb, url, flags, options);
+
     if (r >= 0) {
         if (options)
             mp_avdict_print_unset(demuxer->log, MSGL_TRACE, *options);
         struct nested_stream nest = {
             .id = *pb,
+            .curl_data = curl_data,
         };
         MP_TARRAY_APPEND(priv, priv->nested, priv->num_nested, nest);
     }
@@ -963,13 +978,22 @@ static int nested_io_close2(struct AVFormatContext *s, AVIOContext *pb)
     mp_require(demuxer);
     lavf_priv_t *priv = demuxer->priv;
 
+    MP_UNUSED void *curl_data = NULL;
     for (int n = 0; n < priv->num_nested; n++) {
         if (priv->nested[n].id == pb) {
+            curl_data = priv->nested[n].curl_data;
             MP_TARRAY_REMOVE_AT(priv->nested, priv->num_nested, n);
             break;
         }
     }
 
+#if HAVE_LIBCURL
+    if (curl_data) {
+        mp_curl_avio_close(pb, curl_data);
+        return 0;
+    }
+#endif
+
     return priv->default_io_close2(s, pb);
 }
 
@@ -1350,6 +1374,22 @@ static int demux_open_lavf(demuxer_t *demuxer, enum demux_check check)
         avfc->io_open = block_io_open;
     }
 
+#if HAVE_LIBCURL
+    // When nested HTTP requests are routed through our libcurl backend,
+    // Lavf's HLS demuxer must not try to reuse the URLContext of the previous
+    // request via ff_http_do_new_request2(). Our AVIOContext is not backed by a
+    // URLContext, so ffio_geturlcontext() returns NULL and av_assert0() trips.
+    // Note that our implementation will reuse and multiplex connections.
+    // This will be fixed upstream, but keep compatibility with older versions.
+    if (demuxer->access_references) {
+        av_dict_set(&dopts, "http_persistent", "0", 0);
+        // Actually enable http_multiple, this is basic prefetching logic in
+        // HLS demuxer. We just need to avoid autodetection (-1) which would
+        // be incorrect as our backed is handling everything.
+        av_dict_set(&dopts, "http_multiple", "1", 0);
+    }
+#endif
+
     mp_set_avdict(&dopts, lavfdopts->avopts);
 
     if (av_dict_copy(&priv->av_opts, dopts, 0) < 0) {
@@ -1506,6 +1546,8 @@ static bool demux_lavf_read_packet(struct demuxer *demux,
         av_packet_free(&pkt);
         if (r == AVERROR_EOF)
             return false;
+        if (mp_cancel_test(demux->cancel))
+            return false;
         MP_WARN(demux, "error reading packet: %s.\n", av_err2str(r));
         if (priv->retry_counter >= 10) {
             MP_ERR(demux, "...treating it as fatal error.\n");

meson.build

@@ -184,6 +184,7 @@ sources = files(
 
     ## Streams
     'stream/cookies.c',
+    'stream/network.c',
     'stream/stream.c',
     'stream/stream_avdevice.c',
     'stream/stream_cb.c',
@@ -620,6 +621,13 @@ if darwin
     subdir(join_paths('TOOLS', 'osxbundle'))
 endif
 
+libcurl = dependency('libcurl', version: '>= 7.83.0', required: get_option('libcurl'))
+features += {'libcurl': libcurl.found()}
+if features['libcurl']
+    dependencies += [libcurl]
+    sources += files('stream/stream_curl.c')
+endif
+
 cdda_opt = get_option('cdda').require(
     get_option('gpl'),
     error_message: 'the build is not GPL!',
@@ -1914,6 +1922,7 @@ if features['sdl2-audio'] or features['sdl2-gamepad'] or features['sdl2-video']
 endif
 
 summary({'cocoa': features['cocoa'] and features['swift'],
+         'curl': features['libcurl'],
          'd3d11': features['d3d11'],
          'drm': features['drm'],
          'javascript': features['javascript'],

meson.options

@@ -19,6 +19,7 @@ option('lcms2', type: 'feature', value: 'auto', description: 'LCMS2 support')
 option('libarchive', type: 'feature', value: 'auto', description: 'libarchive wrapper for reading zip files and more')
 option('libavdevice', type: 'feature', value: 'auto', description: 'libavdevice')
 option('libbluray', type: 'feature', value: 'auto', description: 'Bluray support')
+option('libcurl', type: 'feature', value: 'auto', description: 'libcurl-based stream backend')
 option('lua',
     type: 'combo',
     choices: ['lua', 'lua52', 'lua5.2', 'lua-5.2', 'luajit', 'lua51',

options/options.c

@@ -61,6 +61,7 @@ extern const struct m_sub_options tv_params_conf;
 extern const struct m_sub_options stream_bluray_conf;
 extern const struct m_sub_options stream_cdda_conf;
 extern const struct m_sub_options stream_dvb_conf;
+extern const struct m_sub_options mp_network_conf;
 extern const struct m_sub_options stream_lavf_conf;
 extern const struct m_sub_options sws_conf;
 extern const struct m_sub_options zimg_conf;
@@ -94,6 +95,7 @@ extern const struct m_sub_options ao_conf;
 
 extern const struct m_sub_options dvd_conf;
 extern const struct m_sub_options clipboard_conf;
+extern const struct m_sub_options curl_conf;
 
 extern const struct m_sub_options opengl_conf;
 extern const struct m_sub_options vulkan_conf;
@@ -672,6 +674,7 @@ static const m_option_t mp_opts[] = {
 #if HAVE_DVBIN
     {"dvbin", OPT_SUBSTRUCT(stream_dvb_opts, stream_dvb_conf)},
 #endif
+    {"", OPT_SUBSTRUCT(network_opts, mp_network_conf)},
     {"", OPT_SUBSTRUCT(stream_lavf_opts, stream_lavf_conf)},
 
 // ------------------------- a-v sync options --------------------
@@ -924,6 +927,10 @@ static const m_option_t mp_opts[] = {
 
     {"clipboard", OPT_SUBSTRUCT(clipboard_opts, clipboard_conf)},
 
+#if HAVE_LIBCURL
+    {"curl", OPT_SUBSTRUCT(curl_opts, curl_conf)},
+#endif
+
     {"", OPT_SUBSTRUCT(vo, vo_sub_opts)},
     {"", OPT_SUBSTRUCT(demux_opts, demux_conf)},
     {"", OPT_SUBSTRUCT(demux_cache_opts, demux_cache_conf)},

options/options.h

@@ -369,7 +369,8 @@ typedef struct MPOpts {
     struct mp_bluray_opts *stream_bluray_opts;
     struct cdda_opts *stream_cdda_opts;
     struct dvb_opts *stream_dvb_opts;
-    struct lavf_opts *stream_lavf_opts;
+    struct mp_network_opts *network_opts;
+    struct stream_lavf_opts *stream_lavf_opts;
 
     struct demux_rawaudio_opts *demux_rawaudio;
     struct demux_rawvideo_opts *demux_rawvideo;
@@ -394,6 +395,8 @@ typedef struct MPOpts {
 
     struct clipboard_opts *clipboard_opts;
 
+    struct curl_opts *curl_opts;
+
     struct encode_opts *encode_opts;
 
     char *ipc_path;