[build] Automated builds and nightly releases (#6220)

Closes #1839
Authored by: Grub4K, bashonly

Co-authored-by: bashonly <88596187+bashonly@users.noreply.github.com>

.github/workflows/build.yml

@@ -1,74 +1,100 @@
-name: Build
+name: Build Artifacts
-on: workflow_dispatch
+on:
+  workflow_call:
+    inputs:
+      version:
+        required: true
+        type: string
+      channel:
+        required: false
+        default: stable
+        type: string
+      unix:
+        default: true
+        type: boolean
+      linux_arm:
+        default: true
+        type: boolean
+      macos:
+        default: true
+        type: boolean
+      macos_legacy:
+        default: true
+        type: boolean
+      windows:
+        default: true
+        type: boolean
+      windows32:
+        default: true
+        type: boolean
+      meta_files:
+        default: true
+        type: boolean
+
+  workflow_dispatch:
+    inputs:
+      version:
+        description: Version tag (YYYY.MM.DD[.REV])
+        required: true
+        type: string
+      channel:
+        description: Update channel (stable/nightly)
+        required: true
+        default: stable
+        type: string
+      unix:
+        description: yt-dlp, yt-dlp.tar.gz, yt-dlp_linux, yt-dlp_linux.zip
+        default: true
+        type: boolean
+      linux_arm:
+        description: yt-dlp_linux_aarch64, yt-dlp_linux_armv7l
+        default: true
+        type: boolean
+      macos:
+        description: yt-dlp_macos, yt-dlp_macos.zip
+        default: true
+        type: boolean
+      macos_legacy:
+        description: yt-dlp_macos_legacy
+        default: true
+        type: boolean
+      windows:
+        description: yt-dlp.exe, yt-dlp_min.exe, yt-dlp_win.zip
+        default: true
+        type: boolean
+      windows32:
+        description: yt-dlp_x86.exe
+        default: true
+        type: boolean
+      meta_files:
+        description: SHA2-256SUMS, SHA2-512SUMS, _update_spec
+        default: true
+        type: boolean
+
 permissions:
   contents: read
 
 jobs:
-  prepare:
+  unix:
-    permissions:
+    if: inputs.unix
-      contents: write  # for push_release
     runs-on: ubuntu-latest
-    outputs:
-      version_suffix: ${{ steps.version_suffix.outputs.version_suffix }}
-      ytdlp_version: ${{ steps.bump_version.outputs.ytdlp_version }}
-      head_sha: ${{ steps.push_release.outputs.head_sha }}
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-    - uses: actions/setup-python@v4
-      with:
-          python-version: '3.10'
-
-    - name: Set version suffix
-      id: version_suffix
-      env:
-        PUSH_VERSION_COMMIT: ${{ secrets.PUSH_VERSION_COMMIT }}
-      if: "env.PUSH_VERSION_COMMIT == ''"
-      run: echo "version_suffix=$(date -u +"%H%M%S")" >> "$GITHUB_OUTPUT"
-    - name: Bump version
-      id: bump_version
-      run: |
-        python devscripts/update-version.py ${{ steps.version_suffix.outputs.version_suffix }}
-        make issuetemplates
-
-    - name: Push to release
-      id: push_release
-      run: |
-        git config --global user.name github-actions
-        git config --global user.email github-actions@example.com
-        git add -u
-        git commit -m "[version] update" -m "Created by: ${{ github.event.sender.login }}" -m ":ci skip all :ci run dl"
-        git push origin --force ${{ github.event.ref }}:release
-        echo "head_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
-    - name: Update master
-      env:
-        PUSH_VERSION_COMMIT: ${{ secrets.PUSH_VERSION_COMMIT }}
-      if: "env.PUSH_VERSION_COMMIT != ''"
-      run: git push origin ${{ github.event.ref }}
-
-
-  build_unix:
-    needs: prepare
-    runs-on: ubuntu-latest
-
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: "3.10"
       - uses: conda-incubator/setup-miniconda@v2
         with:
           miniforge-variant: Mambaforge
           use-mamba: true
           channels: conda-forge
           auto-update-conda: true
-          activate-environment: ''
+          activate-environment: ""
           auto-activate-base: false
       - name: Install Requirements
         run: |
           sudo apt-get -y install zip pandoc man sed
-          python -m pip install -U pip setuptools wheel twine
+          python -m pip install -U pip setuptools wheel
           python -m pip install -U Pyinstaller -r requirements.txt
           reqs=$(mktemp)
           echo -e 'python=3.10.*\npyinstaller' >$reqs
@@ -77,7 +103,7 @@ jobs:
 
       - name: Prepare
         run: |
-          python devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }}
+          python devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
           python devscripts/make_lazy_extractors.py
       - name: Build Unix platform-independent binary
         run: |
@@ -90,6 +116,8 @@ jobs:
           python pyinst.py --onedir
           (cd ./dist/yt-dlp_linux && zip -r ../yt-dlp_linux.zip .)
           python pyinst.py
+          mv ./dist/yt-dlp_linux ./yt-dlp_linux
+          mv ./dist/yt-dlp_linux.zip ./yt-dlp_linux.zip
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v3
@@ -97,45 +125,15 @@ jobs:
           path: |
             yt-dlp
             yt-dlp.tar.gz
-          dist/yt-dlp_linux
+            yt-dlp_linux
-          dist/yt-dlp_linux.zip
+            yt-dlp_linux.zip
 
-    - name: Build and publish on PyPi
+  linux_arm:
-      env:
+    if: inputs.linux_arm
-        TWINE_USERNAME: __token__
-        TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
-      if: "env.TWINE_PASSWORD != ''"
-      run: |
-        rm -rf dist/*
-        python devscripts/set-variant.py pip -M "You installed yt-dlp with pip or using the wheel from PyPi; Use that to update"
-        python setup.py sdist bdist_wheel
-        twine upload dist/*
-
-    - name: Install SSH private key for Homebrew
-      env:
-        BREW_TOKEN: ${{ secrets.BREW_TOKEN }}
-      if: "env.BREW_TOKEN != ''"
-      uses: yt-dlp/ssh-agent@v0.5.3
-      with:
-          ssh-private-key: ${{ env.BREW_TOKEN }}
-    - name: Update Homebrew Formulae
-      env:
-        BREW_TOKEN: ${{ secrets.BREW_TOKEN }}
-      if: "env.BREW_TOKEN != ''"
-      run: |
-        git clone git@github.com:yt-dlp/homebrew-taps taps/
-        python devscripts/update-formulae.py taps/Formula/yt-dlp.rb "${{ needs.prepare.outputs.ytdlp_version }}"
-        git -C taps/ config user.name github-actions
-        git -C taps/ config user.email github-actions@example.com
-        git -C taps/ commit -am 'yt-dlp: ${{ needs.prepare.outputs.ytdlp_version }}'
-        git -C taps/ push
-
-
-  build_linux_arm:
     permissions:
-      packages: write  # for Creating cache
+      contents: read
+      packages: write # for creating cache
     runs-on: ubuntu-latest
-    needs: prepare
     strategy:
       matrix:
         architecture:
@@ -149,6 +147,9 @@ jobs:
       - name: Virtualized Install, Prepare & Build
         uses: yt-dlp/run-on-arch-action@v2
         with:
+          # Ref: https://github.com/uraimo/run-on-arch-action/issues/55
+          env: |
+            GITHUB_WORKFLOW: build
           githubToken: ${{ github.token }} # To cache image
           arch: ${{ matrix.architecture }}
           distro: ubuntu18.04 # Standalone executable should be built on minimum supported OS
@@ -163,7 +164,7 @@ jobs:
           run: |
             cd repo
             python3.8 -m pip install -U Pyinstaller -r requirements.txt  # Cached version may be out of date
-          python3.8 devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }}
+            python3.8 devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
             python3.8 devscripts/make_lazy_extractors.py
             python3.8 pyinst.py
 
@@ -173,10 +174,9 @@ jobs:
           path: | # run-on-arch-action designates armv7l as armv7
             repo/dist/yt-dlp_linux_${{ (matrix.architecture == 'armv7' && 'armv7l') || matrix.architecture }}
 
-
+  macos:
-  build_macos:
+    if: inputs.macos
     runs-on: macos-11
-    needs: prepare
 
     steps:
       - uses: actions/checkout@v3
@@ -188,7 +188,7 @@ jobs:
 
       - name: Prepare
         run: |
-          /usr/bin/python3 devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }}
+          /usr/bin/python3 devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
           /usr/bin/python3 devscripts/make_lazy_extractors.py
       - name: Build
         run: |
@@ -203,10 +203,9 @@ jobs:
             dist/yt-dlp_macos
             dist/yt-dlp_macos.zip
 
-
+  macos_legacy:
-  build_macos_legacy:
+    if: inputs.macos_legacy
     runs-on: macos-latest
-    needs: prepare
 
     steps:
       - uses: actions/checkout@v3
@@ -229,7 +228,7 @@ jobs:
 
       - name: Prepare
         run: |
-          python3 devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }}
+          python3 devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
           python3 devscripts/make_lazy_extractors.py
       - name: Build
         run: |
@@ -242,16 +241,15 @@ jobs:
           path: |
             dist/yt-dlp_macos_legacy
 
-
+  windows:
-  build_windows:
+    if: inputs.windows
     runs-on: windows-latest
-    needs: prepare
 
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with: # 3.8 is used for Win7 support
-          python-version: '3.8'
+          python-version: "3.8"
       - name: Install Requirements
         run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds
           python -m pip install -U pip setuptools wheel py2exe
@@ -259,7 +257,7 @@ jobs:
 
       - name: Prepare
         run: |
-          python devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }}
+          python devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
           python devscripts/make_lazy_extractors.py
       - name: Build
         run: |
@@ -277,17 +275,16 @@ jobs:
             dist/yt-dlp_min.exe
             dist/yt-dlp_win.zip
 
-
+  windows32:
-  build_windows32:
+    if: inputs.windows32
     runs-on: windows-latest
-    needs: prepare
 
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with: # 3.7 is used for Vista support. See https://github.com/yt-dlp/yt-dlp/issues/390
-          python-version: '3.7'
+          python-version: "3.7"
-          architecture: 'x86'
+          architecture: "x86"
       - name: Install Requirements
         run: |
           python -m pip install -U pip setuptools wheel
@@ -295,7 +292,7 @@ jobs:
 
       - name: Prepare
         run: |
-          python devscripts/update-version.py ${{ needs.prepare.outputs.version_suffix }}
+          python devscripts/update-version.py -c ${{ inputs.channel }} ${{ inputs.version }}
           python devscripts/make_lazy_extractors.py
       - name: Build
         run: |
@@ -307,87 +304,35 @@ jobs:
           path: |
             dist/yt-dlp_x86.exe
 
-
+  meta_files:
-  publish_release:
+    if: inputs.meta_files && always()
-    permissions:
+    needs:
-      contents: write  # for action-gh-release
+      - unix
+      - linux_arm
+      - macos
+      - macos_legacy
+      - windows
+      - windows32
     runs-on: ubuntu-latest
-    needs: [prepare, build_unix, build_linux_arm, build_windows, build_windows32, build_macos, build_macos_legacy]
-
     steps:
-    - uses: actions/checkout@v3
       - uses: actions/download-artifact@v3
 
-    - name: Get Changelog
-      run: |
-        changelog=$(grep -oPz '(?s)(?<=### ${{ needs.prepare.outputs.ytdlp_version }}\n{2}).+?(?=\n{2,3}###)' Changelog.md) || true
-        echo "changelog<<EOF" >> $GITHUB_ENV
-        echo "$changelog" >> $GITHUB_ENV
-        echo "EOF" >> $GITHUB_ENV
-    - name: Make Update spec
-      run: |
-        echo "# This file is used for regulating self-update" >> _update_spec
-        echo "lock 2022.07.18 .+ Python 3.6" >> _update_spec
       - name: Make SHA2-SUMS files
         run: |
-          sha256sum artifact/yt-dlp | awk '{print $1 "  yt-dlp"}' >> SHA2-256SUMS
+          cd ./artifact/
-          sha256sum artifact/yt-dlp.tar.gz | awk '{print $1 "  yt-dlp.tar.gz"}' >> SHA2-256SUMS
+          sha256sum * > ../SHA2-256SUMS
-          sha256sum artifact/yt-dlp.exe | awk '{print $1 "  yt-dlp.exe"}' >> SHA2-256SUMS
+          sha512sum * > ../SHA2-512SUMS
-          sha256sum artifact/yt-dlp_win.zip | awk '{print $1 "  yt-dlp_win.zip"}' >> SHA2-256SUMS
-          sha256sum artifact/yt-dlp_min.exe | awk '{print $1 "  yt-dlp_min.exe"}' >> SHA2-256SUMS
-          sha256sum artifact/yt-dlp_x86.exe | awk '{print $1 "  yt-dlp_x86.exe"}' >> SHA2-256SUMS
-          sha256sum artifact/yt-dlp_macos | awk '{print $1 "  yt-dlp_macos"}' >> SHA2-256SUMS
-          sha256sum artifact/yt-dlp_macos.zip | awk '{print $1 "  yt-dlp_macos.zip"}' >> SHA2-256SUMS
-          sha256sum artifact/yt-dlp_macos_legacy | awk '{print $1 "  yt-dlp_macos_legacy"}' >> SHA2-256SUMS
-          sha256sum artifact/yt-dlp_linux_armv7l | awk '{print $1 "  yt-dlp_linux_armv7l"}' >> SHA2-256SUMS
-          sha256sum artifact/yt-dlp_linux_aarch64 | awk '{print $1 "  yt-dlp_linux_aarch64"}' >> SHA2-256SUMS
-          sha256sum artifact/dist/yt-dlp_linux | awk '{print $1 "  yt-dlp_linux"}' >> SHA2-256SUMS
-          sha256sum artifact/dist/yt-dlp_linux.zip | awk '{print $1 "  yt-dlp_linux.zip"}' >> SHA2-256SUMS
-          sha512sum artifact/yt-dlp | awk '{print $1 "  yt-dlp"}' >> SHA2-512SUMS
-          sha512sum artifact/yt-dlp.tar.gz | awk '{print $1 "  yt-dlp.tar.gz"}' >> SHA2-512SUMS
-          sha512sum artifact/yt-dlp.exe | awk '{print $1 "  yt-dlp.exe"}' >> SHA2-512SUMS
-          sha512sum artifact/yt-dlp_win.zip | awk '{print $1 "  yt-dlp_win.zip"}' >> SHA2-512SUMS
-          sha512sum artifact/yt-dlp_min.exe | awk '{print $1 "  yt-dlp_min.exe"}' >> SHA2-512SUMS
-          sha512sum artifact/yt-dlp_x86.exe | awk '{print $1 "  yt-dlp_x86.exe"}' >> SHA2-512SUMS
-          sha512sum artifact/yt-dlp_macos | awk '{print $1 "  yt-dlp_macos"}' >> SHA2-512SUMS
-          sha512sum artifact/yt-dlp_macos.zip | awk '{print $1 "  yt-dlp_macos.zip"}' >> SHA2-512SUMS
-          sha512sum artifact/yt-dlp_macos_legacy | awk '{print $1 "  yt-dlp_macos_legacy"}' >> SHA2-512SUMS
-          sha512sum artifact/yt-dlp_linux_armv7l | awk '{print $1 "  yt-dlp_linux_armv7l"}' >> SHA2-512SUMS
-          sha512sum artifact/yt-dlp_linux_aarch64 | awk '{print $1 "  yt-dlp_linux_aarch64"}' >> SHA2-512SUMS
-          sha512sum artifact/dist/yt-dlp_linux | awk '{print $1 "  yt-dlp_linux"}' >> SHA2-512SUMS
-          sha512sum artifact/dist/yt-dlp_linux.zip | awk '{print $1 "  yt-dlp_linux.zip"}' >> SHA2-512SUMS
 
-    - name: Publish Release
+      - name: Make Update spec
-      uses: yt-dlp/action-gh-release@v1
+        run: |
+          cat >> _update_spec << EOF
+          # This file is used for regulating self-update
+          lock 2022.08.18.36 .+ Python 3.6
+          EOF
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
         with:
-        tag_name: ${{ needs.prepare.outputs.ytdlp_version }}
+          path: |
-        name: yt-dlp ${{ needs.prepare.outputs.ytdlp_version }}
+            SHA*SUMS*
-        target_commitish: ${{ needs.prepare.outputs.head_sha }}
-        body: |
-          #### [A description of the various files]((https://github.com/yt-dlp/yt-dlp#release-files)) are in the README
-
-          ---
-          <details open><summary><h3>Changelog</summary>
-          <p>
-
-          ${{ env.changelog }}
-
-          </p>
-          </details>
-        files: |
-          SHA2-256SUMS
-          SHA2-512SUMS
-          artifact/yt-dlp
-          artifact/yt-dlp.tar.gz
-          artifact/yt-dlp.exe
-          artifact/yt-dlp_win.zip
-          artifact/yt-dlp_min.exe
-          artifact/yt-dlp_x86.exe
-          artifact/yt-dlp_macos
-          artifact/yt-dlp_macos.zip
-          artifact/yt-dlp_macos_legacy
-          artifact/yt-dlp_linux_armv7l
-          artifact/yt-dlp_linux_aarch64
-          artifact/dist/yt-dlp_linux
-          artifact/dist/yt-dlp_linux.zip
             _update_spec

.github/workflows/publish.yml

@@ -0,0 +1,80 @@
+name: Publish
+on:
+  workflow_call:
+    inputs:
+      nightly:
+        default: false
+        required: false
+        type: boolean
+      version:
+        required: true
+        type: string
+      target_commitish:
+        required: true
+        type: string
+    secrets:
+      ARCHIVE_REPO_TOKEN:
+        required: false
+
+permissions:
+  contents: write
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: actions/download-artifact@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Generate release notes
+        run: |
+          cat >> ./RELEASE_NOTES << EOF
+          #### A description of the various files are in the [README](https://github.com/yt-dlp/yt-dlp#release-files)
+          ---
+          <details><summary><h3>Changelog</h3></summary>
+          $(python ./devscripts/make_changelog.py -vv)
+          </details>
+          EOF
+          echo "**This is an automated nightly pre-release build**" >> ./PRERELEASE_NOTES
+          cat ./RELEASE_NOTES >> ./PRERELEASE_NOTES
+          echo "Generated from: https://github.com/${{ github.repository }}/commit/${{ inputs.target_commitish }}" >> ./ARCHIVE_NOTES
+          cat ./RELEASE_NOTES >> ./ARCHIVE_NOTES
+
+      - name: Archive nightly release
+        env:
+          GH_TOKEN: ${{ secrets.ARCHIVE_REPO_TOKEN }}
+          GH_REPO: ${{ vars.ARCHIVE_REPO }}
+        if: |
+          inputs.nightly && env.GH_TOKEN != '' && env.GH_REPO != ''
+        run: |
+          gh release create \
+            --notes-file ARCHIVE_NOTES \
+            --title "Build ${{ inputs.version }}" \
+            ${{ inputs.version }} \
+            artifact/*
+
+      - name: Prune old nightly release
+        if: inputs.nightly
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh release delete --yes --cleanup-tag "nightly" || true
+          git tag --delete "nightly" || true
+          sleep 5  # Enough time to cover deletion race condition
+
+      - name: Publish release${{ inputs.nightly && ' (nightly)' || '' }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh release create \
+            --notes-file ${{ inputs.nightly && 'PRE' || '' }}RELEASE_NOTES \
+            --target ${{ inputs.target_commitish }} \
+            --title "yt-dlp ${{ inputs.nightly && 'nightly ' || '' }}${{ inputs.version }}" \
+            ${{ inputs.nightly && '--prerelease "nightly"' || inputs.version }} \
+            artifact/*

.github/workflows/release-nightly.yml

@@ -0,0 +1,49 @@
+name: Release (nightly)
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - "**.py"
+      - "!yt_dlp/version.py"
+concurrency:
+  group: release-nightly
+  cancel-in-progress: true
+permissions:
+  contents: read
+
+jobs:
+  prepare:
+    if: vars.BUILD_NIGHTLY != ''
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.get_version.outputs.version }}
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Get version
+        id: get_version
+        run: |
+          python devscripts/update-version.py "$(date -u +"%H%M%S")" | grep -Po "version=\d+(\.\d+){3}" >> "$GITHUB_OUTPUT"
+
+  build:
+    needs: prepare
+    uses: ./.github/workflows/build.yml
+    with:
+      version: ${{ needs.prepare.outputs.version }}
+      channel: nightly
+    permissions:
+      contents: read
+      packages: write # For package cache
+
+  publish:
+    needs: [prepare, build]
+    uses: ./.github/workflows/publish.yml
+    secrets:
+      ARCHIVE_REPO_TOKEN: ${{ secrets.ARCHIVE_REPO_TOKEN }}
+    permissions:
+      contents: write
+    with:
+      nightly: true
+      version: ${{ needs.prepare.outputs.version }}
+      target_commitish: ${{ github.sha }}

.github/workflows/release.yml

@@ -0,0 +1,125 @@
+name: Release
+on: workflow_dispatch
+permissions:
+  contents: read
+
+jobs:
+  prepare:
+    permissions:
+      contents: write
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.update_version.outputs.version }}
+      head_sha: ${{ steps.push_release.outputs.head_sha }}
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Update version
+        id: update_version
+        run: |
+          python devscripts/update-version.py ${{ vars.PUSH_VERSION_COMMIT == '' && '"$(date -u +"%H%M%S")"' || '' }} | \
+            grep -Po "version=\d+\.\d+\.\d+(\.\d+)?" >> "$GITHUB_OUTPUT"
+
+      - name: Update documentation
+        run: |
+          make doc
+          sed '/### /Q' Changelog.md >> ./CHANGELOG
+          echo '### ${{ steps.update_version.outputs.version }}' >> ./CHANGELOG
+          python ./devscripts/make_changelog.py -vv -c >> ./CHANGELOG
+          echo >> ./CHANGELOG
+          grep -Poz '(?s)### \d+\.\d+\.\d+.+' 'Changelog.md' | head -n -1 >> ./CHANGELOG
+          cat ./CHANGELOG > Changelog.md
+
+      - name: Push to release
+        id: push_release
+        run: |
+          git config --global user.name github-actions
+          git config --global user.email github-actions@example.com
+          git add -u
+          git commit -m "Release ${{ steps.update_version.outputs.version }}" \
+            -m "Created by: ${{ github.event.sender.login }}" -m ":ci skip all :ci run dl"
+          git push origin --force ${{ github.event.ref }}:release
+          echo "head_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+
+      - name: Update master
+        if: vars.PUSH_VERSION_COMMIT != ''
+        run: git push origin ${{ github.event.ref }}
+
+  publish_pypi_homebrew:
+    needs: prepare
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Install Requirements
+        run: |
+          python -m pip install -U pip setuptools wheel twine
+          python -m pip install -U -r requirements.txt
+
+      - name: Prepare
+        run: |
+          python devscripts/update-version.py ${{ needs.prepare.outputs.version }}
+          python devscripts/make_lazy_extractors.py
+
+      - name: Build and publish on PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+        if: env.TWINE_PASSWORD != ''
+        run: |
+          rm -rf dist/*
+          python devscripts/set-variant.py pip -M "You installed yt-dlp with pip or using the wheel from PyPi; Use that to update"
+          python setup.py sdist bdist_wheel
+          twine upload dist/*
+
+      - name: Checkout Homebrew repository
+        env:
+          BREW_TOKEN: ${{ secrets.BREW_TOKEN }}
+          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+        if: env.BREW_TOKEN != '' && env.PYPI_TOKEN != ''
+        uses: actions/checkout@v3
+        with:
+          repository: yt-dlp/homebrew-taps
+          path: taps
+          ssh-key: ${{ secrets.BREW_TOKEN }}
+
+      - name: Update Homebrew Formulae
+        env:
+          BREW_TOKEN: ${{ secrets.BREW_TOKEN }}
+          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+        if: env.BREW_TOKEN != '' && env.PYPI_TOKEN != ''
+        run: |
+          python devscripts/update-formulae.py taps/Formula/yt-dlp.rb "${{ needs.prepare.outputs.version }}"
+          git -C taps/ config user.name github-actions
+          git -C taps/ config user.email github-actions@example.com
+          git -C taps/ commit -am 'yt-dlp: ${{ needs.prepare.outputs.version }}'
+          git -C taps/ push
+
+  build:
+    needs: prepare
+    uses: ./.github/workflows/build.yml
+    with:
+      version: ${{ needs.prepare.outputs.version }}
+    permissions:
+      contents: read
+      packages: write # For package cache
+
+  publish:
+    needs: [prepare, build]
+    uses: ./.github/workflows/publish.yml
+    permissions:
+      contents: write
+    with:
+      version: ${{ needs.prepare.outputs.version }}
+      target_commitish: ${{ needs.prepare.outputs.head_sha }}

Changelog.md

@@ -1,13 +1,7 @@
 # Changelog
 
 <!--
-# Instuctions for creating release
+# To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master
-
-* Run `make doc`
-* Update Changelog.md and CONTRIBUTORS
-* Change "Based on ytdl" version in Readme.md if needed
-* Commit as `Release <version>` and push to master
-* Dispatch the workflow https://github.com/yt-dlp/yt-dlp/actions/workflows/build.yml on master
 -->
 
 ### 2023.02.17

README.md

@@ -318,7 +318,8 @@ If you wish to build it anyway, install Python and py2exe, and then simply run `
 
 Note: See their `--help` for more info.
 
-You can also fork the project on GitHub and run your fork's [build workflow](.github/workflows/build.yml) to automatically build a full release
+### Forking the project
+If you fork the project on GitHub, you can run your fork's [build workflow](.github/workflows/build.yml) to automatically build the selected version(s) as artifacts. Alternatively, you can run the [release workflow](.github/workflows/release.yml) or enable the [nightly workflow](.github/workflows/release-nightly.yml) to create full (pre-)releases.
 
 # USAGE AND OPTIONS
 
@@ -460,9 +461,8 @@ You can also fork the project on GitHub and run your fork's [build workflow](.gi
     --date DATE                     Download only videos uploaded on this date.
                                     The date can be "YYYYMMDD" or in the format 
                                     [now|today|yesterday][-N[day|week|month|year]].
-                                    E.g. "--date today-2weeks" downloads
+                                    E.g. "--date today-2weeks" downloads only
-                                    only videos uploaded on the same day two
+                                    videos uploaded on the same day two weeks ago
-                                    weeks ago
     --datebefore DATE               Download only videos uploaded on or before
                                     this date. The date formats accepted is the
                                     same as --date

devscripts/make_readme.py

@@ -45,7 +45,7 @@ switch_col_width = len(re.search(r'(?m)^\s{5,}', options).group())
 delim = f'\n{" " * switch_col_width}'
 
 PATCHES = (
-    (   # Standardize update message
+    (   # Standardize `--update` message
         r'(?m)^(    -U, --update\s+).+(\n    \s.+)*$',
         r'\1Update this program to the latest version',
     ),
@@ -53,6 +53,12 @@ PATCHES = (
         r'(?m)^  (\w.+\n)(    (?=\w))?',
         r'## \1'
     ),
+    (   # Fixup `--date` formatting
+        rf'(?m)(    --date DATE.+({delim}[^\[]+)*)\[.+({delim}.+)*$',
+        (rf'\1[now|today|yesterday][-N[day|week|month|year]].{delim}'
+         f'E.g. "--date today-2weeks" downloads only{delim}'
+         'videos uploaded on the same day two weeks ago'),
+    ),
     (   # Do not split URLs
         rf'({delim[:-1]})? (?P<label>\[\S+\] )?(?P<url>https?({delim})?:({delim})?/({delim})?/(({delim})?\S+)+)\s',
         lambda mobj: ''.join((delim, mobj.group('label') or '', re.sub(r'\s+', '', mobj.group('url')), '\n'))
@@ -72,6 +78,10 @@ PATCHES = (
         r'(?m)^(\s{4}-.{%d})(%s)' % (switch_col_width - 6, delim),
         r'\1 '
     ),
+    (   # Replace brackets with a Markdown link
+        r'SponsorBlock API \((http.+)\)',
+        r'[SponsorBlock API](\1)'
+    ),
 )
 
 readme = read_file(README_FILE)

devscripts/update-version.py

@@ -7,6 +7,7 @@ import sys
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
+import argparse
 import contextlib
 import subprocess
 import sys
@@ -15,7 +16,8 @@ from datetime import datetime
 from devscripts.utils import read_version, write_file
 
 
-def get_new_version(revision):
+def get_new_version(version, revision):
+    if not version:
         version = datetime.utcnow().strftime('%Y.%m.%d')
 
     if revision:
@@ -30,27 +32,41 @@ def get_new_version(revision):
 
 def get_git_head():
     with contextlib.suppress(Exception):
-        sp = subprocess.Popen(['git', 'rev-parse', '--short', 'HEAD'], stdout=subprocess.PIPE)
+        return subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip() or None
-        return sp.communicate()[0].decode().strip() or None
 
 
-VERSION = get_new_version((sys.argv + [''])[1])
+VERSION_TEMPLATE = '''\
-GIT_HEAD = get_git_head()
-
-VERSION_FILE = f'''\
 # Autogenerated by devscripts/update-version.py
 
-__version__ = {VERSION!r}
+__version__ = {version!r}
 
-RELEASE_GIT_HEAD = {GIT_HEAD!r}
+RELEASE_GIT_HEAD = {git_head!r}
 
 VARIANT = None
 
 UPDATE_HINT = None
+
+CHANNEL = "{channel!r}"
 '''
 
-write_file('yt_dlp/version.py', VERSION_FILE)
+if __name__ == '__main__':
-github_output = os.getenv('GITHUB_OUTPUT')
+    parser = argparse.ArgumentParser(description='Update the version.py file')
-if github_output:
+    parser.add_argument(
-    write_file(github_output, f'ytdlp_version={VERSION}\n', 'a')
+        '-c', '--channel', choices=['stable', 'nightly'], default='stable',
-print(f'\nVersion = {VERSION}, Git HEAD = {GIT_HEAD}')
+        help='Select update channel (default: %(default)s)')
+    parser.add_argument(
+        '-o', '--output', default='yt_dlp/version.py',
+        help='The output file to write to (default: %(default)s)')
+    parser.add_argument(
+        'version', nargs='?', default=None,
+        help='A version or revision to use instead of generating one')
+    args = parser.parse_args()
+
+    git_head = get_git_head()
+    version = (
+        args.version if args.version and '.' in args.version
+        else get_new_version(None, args.version))
+    write_file(args.output, VERSION_TEMPLATE.format(
+        version=version, git_head=git_head, channel=args.channel))
+
+    print(f'version={version} ({args.channel}), head={git_head}')

yt_dlp/YoutubeDL.py

@@ -150,7 +150,7 @@ from .utils import (
     write_json_file,
     write_string,
 )
-from .version import RELEASE_GIT_HEAD, VARIANT, __version__
+from .version import CHANNEL, RELEASE_GIT_HEAD, VARIANT, __version__
 
 if compat_os_name == 'nt':
     import ctypes
@@ -3768,8 +3768,8 @@ class YoutubeDL:
         klass = type(self)
         write_debug(join_nonempty(
             f'{"yt-dlp" if REPOSITORY == "yt-dlp/yt-dlp" else REPOSITORY} version',
-            __version__,
+            __version__ + {'stable': '', 'nightly': '*'}.get(CHANNEL, f' <{CHANNEL}>'),
-            f'[{RELEASE_GIT_HEAD}]' if RELEASE_GIT_HEAD else '',
+            f'[{RELEASE_GIT_HEAD[:9]}]' if RELEASE_GIT_HEAD else '',
             '' if source == 'unknown' else f'({source})',
             '' if _IN_CLI else 'API' if klass == YoutubeDL else f'API:{self.__module__}.{klass.__qualname__}',
             delim=' '))