From aec3cc321894ef564f70db5020207f3dd23c0b93 Mon Sep 17 00:00:00 2001 From: coletdjnz Date: Sun, 14 Jul 2024 18:20:03 +1200 Subject: [PATCH] [networking] Add support for zstandard content-encoding Supported by urllib/requests/curl_cffi Authored-by: coletdjnz --- pyproject.toml | 3 ++ test/test_networking.py | 78 +++++++++++++++++++++++++++++++-- yt_dlp/dependencies/__init__.py | 4 ++ yt_dlp/networking/_curlcffi.py | 2 +- yt_dlp/networking/_requests.py | 10 +---- yt_dlp/networking/_urllib.py | 14 +++++- 6 files changed, 97 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 54755da48d..8a35d62962 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,9 @@ curl-cffi = [ "curl-cffi==0.5.10; os_name=='nt' and implementation_name=='cpython'", "curl-cffi>=0.5.10,!=0.6.*,<0.8; os_name!='nt' and implementation_name=='cpython'", ] +zstd = [ + "zstandard>=0.22.0", +] secretstorage = [ "cffi", "secretstorage", diff --git a/test/test_networking.py b/test/test_networking.py index 826f11a561..8bf9d68da0 100644 --- a/test/test_networking.py +++ b/test/test_networking.py @@ -2,6 +2,7 @@ # Allow direct execution import os +import re import sys import pytest @@ -36,7 +37,7 @@ from test.helper import ( verify_address_availability, ) from yt_dlp.cookies import YoutubeDLCookieJar -from yt_dlp.dependencies import brotli, curl_cffi, requests, urllib3 +from yt_dlp.dependencies import brotli, curl_cffi, requests, urllib3, zstandard from yt_dlp.networking import ( HEADRequest, PUTRequest, @@ -62,7 +63,7 @@ from yt_dlp.networking.impersonate import ( ImpersonateTarget, ) from yt_dlp.utils import YoutubeDLError -from yt_dlp.utils._utils import _YDLLogger as FakeLogger +from yt_dlp.utils._utils import _YDLLogger as FakeLogger, int_or_none from yt_dlp.utils.networking import HTTPHeaderDict, std_headers TEST_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -217,6 +218,7 @@ class HTTPTestRequestHandler(http.server.BaseHTTPRequestHandler): self.end_headers() elif self.path == '/content-encoding': encodings = self.headers.get('ytdl-encoding', '') + content_encoding_header = self.headers.get('ytdl-encoding-header', encodings) payload = b'