[build,update] Add GNU-style SHA512 and prepare updater for simlar SHA256 (#383)

Authored by: nihil-admirari <50202386+nihil-admirari@users.noreply.github.com>

Related: #385

.github/workflows/build.yml

@@ -12,7 +12,8 @@ jobs:
     outputs:
       ytdlp_version: ${{ steps.bump_version.outputs.ytdlp_version }}
       upload_url: ${{ steps.create_release.outputs.upload_url }}
-      sha2_unix: ${{ steps.sha2_file.outputs.sha2_unix }}
+      sha256_unix: ${{ steps.sha256_file.outputs.sha256_unix }}
+      sha512_unix: ${{ steps.sha512_file.outputs.sha512_unix }}
 
     steps:
     - uses: actions/checkout@v2
@@ -53,8 +54,11 @@ jobs:
         asset_name: yt-dlp
         asset_content_type: application/octet-stream
     - name: Get SHA2-256SUMS for yt-dlp
-      id: sha2_file
-      run: echo "::set-output name=sha2_unix::$(sha256sum yt-dlp | awk '{print $1}')"
+      id: sha256_file
+      run: echo "::set-output name=sha256_unix::$(sha256sum yt-dlp | awk '{print $1}')"
+    - name: Get SHA2-512SUMS for yt-dlp
+      id: sha512_file
+      run: echo "::set-output name=sha512_unix::$(sha512sum yt-dlp | awk '{print $1}')"
     - name: Install dependencies for pypi
       env:
         PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
@@ -77,7 +81,8 @@ jobs:
     needs: build_unix
 
     outputs:
-      sha2_windows: ${{ steps.sha2_file_win.outputs.sha2_windows }}
+      sha256_windows: ${{ steps.sha256_file_win.outputs.sha256_windows }}
+      sha512_windows: ${{ steps.sha512_file_win.outputs.sha512_windows }}
 
     steps:
     - uses: actions/checkout@v2
@@ -107,15 +112,19 @@ jobs:
         asset_name: yt-dlp.exe
         asset_content_type: application/vnd.microsoft.portable-executable
     - name: Get SHA2-256SUMS for yt-dlp.exe
-      id: sha2_file_win
-      run: echo "::set-output name=sha2_windows::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA256).Hash.ToLower())"
+      id: sha256_file_win
+      run: echo "::set-output name=sha256_windows::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA256).Hash.ToLower())"
+    - name: Get SHA2-512SUMS for yt-dlp.exe
+      id: sha512_file_win
+      run: echo "::set-output name=sha512_windows::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA512).Hash.ToLower())"
 
   build_windows32:
     runs-on: windows-latest
     needs: [build_unix, build_windows]
 
     outputs:
-      sha2_windows32: ${{ steps.sha2_file_win32.outputs.sha2_windows32 }}
+      sha256_windows32: ${{ steps.sha256_file_win32.outputs.sha256_windows32 }}
+      sha512_windows32: ${{ steps.sha512_file_win32.outputs.sha512_windows32 }}
 
     steps:
     - uses: actions/checkout@v2
@@ -146,8 +155,11 @@ jobs:
         asset_name: yt-dlp_x86.exe
         asset_content_type: application/vnd.microsoft.portable-executable
     - name: Get SHA2-256SUMS for yt-dlp_x86.exe
-      id: sha2_file_win32
-      run: echo "::set-output name=sha2_windows32::$((Get-FileHash dist\yt-dlp_x86.exe -Algorithm SHA256).Hash.ToLower())"
+      id: sha256_file_win32
+      run: echo "::set-output name=sha256_windows32::$((Get-FileHash dist\yt-dlp_x86.exe -Algorithm SHA256).Hash.ToLower())"
+    - name: Get SHA2-512SUMS for yt-dlp_x86.exe
+      id: sha512_file_win32
+      run: echo "::set-output name=sha512_windows32::$((Get-FileHash dist\yt-dlp_x86.exe -Algorithm SHA512).Hash.ToLower())"
 
   finish:
     runs-on: ubuntu-latest
@@ -156,15 +168,15 @@ jobs:
     steps:
     - name: Make SHA2-256SUMS file
       env:
-        SHA2_WINDOWS: ${{ needs.build_windows.outputs.sha2_windows }}
-        SHA2_WINDOWS32: ${{ needs.build_windows32.outputs.sha2_windows32 }}
-        SHA2_UNIX: ${{ needs.build_unix.outputs.sha2_unix }}
+        SHA256_WINDOWS: ${{ needs.build_windows.outputs.sha256_windows }}
+        SHA256_WINDOWS32: ${{ needs.build_windows32.outputs.sha256_windows32 }}
+        SHA256_UNIX: ${{ needs.build_unix.outputs.sha256_unix }}
         YTDLP_VERSION: ${{ needs.build_unix.outputs.ytdlp_version }}
       run: |
-        echo "version: ${{ env.YTDLP_VERSION }}" >> SHA2-256SUMS
-        echo "yt-dlp.exe:${{ env.SHA2_WINDOWS }}" >> SHA2-256SUMS
-        echo "yt-dlp_x86.exe:${{ env.SHA2_WINDOWS32 }}" >> SHA2-256SUMS
-        echo "yt-dlp:${{ env.SHA2_UNIX }}" >> SHA2-256SUMS
+        echo "version:${{ env.YTDLP_VERSION }}" >> SHA2-256SUMS
+        echo "yt-dlp.exe:${{ env.SHA256_WINDOWS }}" >> SHA2-256SUMS
+        echo "yt-dlp_x86.exe:${{ env.SHA256_WINDOWS32 }}" >> SHA2-256SUMS
+        echo "yt-dlp:${{ env.SHA256_UNIX }}" >> SHA2-256SUMS
     - name: Upload 256SUMS file
       id: upload-sums
       uses: actions/upload-release-asset@v1
@@ -175,3 +187,22 @@ jobs:
         asset_path: ./SHA2-256SUMS
         asset_name: SHA2-256SUMS
         asset_content_type: text/plain
+    - name: Make SHA2-512SUMS file
+      env:
+        SHA512_WINDOWS: ${{ needs.build_windows.outputs.sha512_windows }}
+        SHA512_WINDOWS32: ${{ needs.build_windows32.outputs.sha512_windows32 }}
+        SHA512_UNIX: ${{ needs.build_unix.outputs.sha512_unix }}
+      run: |
+        echo "${{ env.SHA512_WINDOWS }}  yt-dlp.exe" >> SHA2-512SUMS
+        echo "${{ env.SHA512_WINDOWS32 }}  yt-dlp_x86.exe" >> SHA2-512SUMS
+        echo "${{ env.SHA512_UNIX }}  yt-dlp" >> SHA2-512SUMS
+    - name: Upload 512SUMS file
+      id: upload-512sums
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ needs.build_unix.outputs.upload_url }}
+        asset_path: ./SHA2-512SUMS
+        asset_name: SHA2-512SUMS
+        asset_content_type: text/plain

yt_dlp/update.py

@@ -135,15 +135,19 @@ def run_update(ydl):
         return next((i for i in version_info['assets'] if i['name'] == 'yt-dlp%s' % label), {})
 
     def get_sha256sum(bin_or_exe, version):
-        label = version_labels['%s_%s' % (bin_or_exe, version)]
+        filename = 'yt-dlp%s' % version_labels['%s_%s' % (bin_or_exe, version)]
         urlh = next(
             (i for i in version_info['assets'] if i['name'] in ('SHA2-256SUMS')),
             {}).get('browser_download_url')
         if not urlh:
             return None
         hash_data = ydl._opener.open(urlh).read().decode('utf-8')
-        hashes = list(map(lambda x: x.split(':'), hash_data.splitlines()))
-        return next((i[1] for i in hashes if i[0] == 'yt-dlp%s' % label), None)
+        if hash_data.startswith('version:'):
+            # Old colon-separated hash file
+            return dict(ln.split(':') for ln in hash_data.splitlines()).get(filename)
+        else:
+            # GNU-style hash file
+            return dict(ln.split()[::-1] for ln in hash_data.splitlines()).get(filename)
 
     if not os.access(filename, os.W_OK):
         return report_error('no write permissions on %s' % filename, expected=True)
@@ -220,7 +224,9 @@ def run_update(ydl):
             return report_error('unable to download latest version', True)
 
         expected_sum = get_sha256sum('zip', '3')
-        if expected_sum and hashlib.sha256(newcontent).hexdigest() != expected_sum:
+        if not expected_sum:
+            ydl.report_warning('no hash information found for the release')
+        elif hashlib.sha256(newcontent).hexdigest() != expected_sum:
             return report_error('unable to verify the new zip', True)
 
         try: