diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md index 352263789..8bf96cb24 100644 --- a/.github/ISSUE_TEMPLATE/1_broken_site.md +++ b/.github/ISSUE_TEMPLATE/1_broken_site.md @@ -18,7 +18,7 @@ title: '' - [ ] I'm reporting a broken site support -- [ ] I've verified that I'm running youtube-dl version **2020.09.14** +- [ ] I've verified that I'm running youtube-dl version **2020.11.01** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped - [ ] I've searched the bugtracker for similar issues including closed ones @@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v < [debug] User config: [] [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj'] [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251 - [debug] youtube-dl version 2020.09.14 + [debug] youtube-dl version 2020.11.01 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4 [debug] Proxy map: {} diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md index fa6509be3..61c005e91 100644 --- a/.github/ISSUE_TEMPLATE/2_site_support_request.md +++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md @@ -19,7 +19,7 @@ labels: 'site-support-request' - [ ] I'm reporting a new site support request -- [ ] I've verified that I'm running youtube-dl version **2020.09.14** +- [ ] I've verified that I'm running youtube-dl version **2020.11.01** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that none of provided URLs violate any copyrights - [ ] I've searched the bugtracker for similar site support requests including closed ones diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md index 70b0f2f19..c7eabe344 100644 --- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md +++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md @@ -18,13 +18,13 @@ title: '' - [ ] I'm reporting a site feature request -- [ ] I've verified that I'm running youtube-dl version **2020.09.14** +- [ ] I've verified that I'm running youtube-dl version **2020.11.01** - [ ] I've searched the bugtracker for similar site feature requests including closed ones diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md index ec17e4a33..446b82e64 100644 --- a/.github/ISSUE_TEMPLATE/4_bug_report.md +++ b/.github/ISSUE_TEMPLATE/4_bug_report.md @@ -18,7 +18,7 @@ title: '' - [ ] I'm reporting a broken site support issue -- [ ] I've verified that I'm running youtube-dl version **2020.09.14** +- [ ] I've verified that I'm running youtube-dl version **2020.11.01** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped - [ ] I've searched the bugtracker for similar bug reports including closed ones @@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v < [debug] User config: [] [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj'] [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251 - [debug] youtube-dl version 2020.09.14 + [debug] youtube-dl version 2020.11.01 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4 [debug] Proxy map: {} diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md index 6ac963206..69657448a 100644 --- a/.github/ISSUE_TEMPLATE/5_feature_request.md +++ b/.github/ISSUE_TEMPLATE/5_feature_request.md @@ -19,13 +19,13 @@ labels: 'request' - [ ] I'm reporting a feature request -- [ ] I've verified that I'm running youtube-dl version **2020.09.14** +- [ ] I've verified that I'm running youtube-dl version **2020.11.01** - [ ] I've searched the bugtracker for similar feature requests including closed ones diff --git a/ChangeLog b/ChangeLog index 4143ec2fb..950042345 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,55 @@ +version 2020.11.01 + +Core +* [utils] Don't attempt to coerce JS strings to numbers in js_to_json (#26851) +* [downloader/http] Properly handle missing message in SSLError (#26646) +* [downloader/http] Fix access to not yet opened stream in retry + +Extractors +* [youtube] Fix JS player URL extraction +* [ytsearch] Fix extraction (#26920) +* [afreecatv] Fix typo (#26970) +* [23video] Relax URL regular expression (#26870) ++ [ustream] Add support for video.ibm.com (#26894) +* [iqiyi] Fix typo (#26884) ++ [expressen] Add support for di.se (#26670) +* [iprima] Improve video id extraction (#26507, #26494) + +version 2020.10.31 + +Core ++ [release] Fix release tarball not being properly generated +* [readme] Fix download link for Unix users + +Extractors +* [youtube] Re-add age restriction test (#14) + +version 2020.10.30 + +Extractors +* [youtube] Fix "Unable to extract JS player URL" (#12) + +version 2020.09.20 + +Core +* [extractor/common] Relax interaction count extraction in _json_ld ++ [extractor/common] Extract author as uploader for VideoObject in _json_ld +* [downloader/hls] Fix incorrect end byte in Range HTTP header for + media segments with EXT-X-BYTERANGE (#14748, #24512) +* [extractor/common] Handle ssl.CertificateError in _request_webpage (#26601) +* [downloader/http] Improve timeout detection when reading block of data + (#10935) +* [downloader/http] Retry download when urlopen times out (#10935, #26603) + +Extractors +* [redtube] Extend URL regular expression (#26506) +* [twitch] Refactor +* [twitch:stream] Switch to GraphQL and fix reruns (#26535) ++ [telequebec] Add support for brightcove videos (#25833) +* [pornhub] Extract metadata from JSON-LD (#26614) +* [pornhub] Fix view count extraction (#26621, #26614) + + version 2020.09.14 Core diff --git a/LICENSE b/LICENSE index 68a49daad..d0df34741 100644 --- a/LICENSE +++ b/LICENSE @@ -1,3 +1,13 @@ +The authors of this software recognize that most technologies can be +used for lawful and unlawful purposes. The authors request that this +software not be used to violate any local laws. The authors recognize +that this is public domain code, that they have no real power +to dictate how it is used other than a polite non-binding request, +and therefore that they cannot be held responsible for what anyone +chooses to do with it. + +--- + This is free and unencumbered software released into the public domain. Anyone is free to copy, modify, publish, use, compile, sell, or diff --git a/Makefile b/Makefile index 3e17365b8..e57ab9b39 100644 --- a/Makefile +++ b/Makefile @@ -117,8 +117,9 @@ _EXTRACTOR_FILES = $(shell find youtube_dl/extractor -iname '*.py' -and -not -in youtube_dl/extractor/lazy_extractors.py: devscripts/make_lazy_extractors.py devscripts/lazy_load_template.py $(_EXTRACTOR_FILES) $(PYTHON) devscripts/make_lazy_extractors.py $@ -youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish ChangeLog AUTHORS - @tar -czf youtube-dl.tar.gz --transform "s|^|youtube-dl/|" --owner 0 --group 0 \ +VERSION = $(shell grep version youtube_dl/version.py |cut -f2 -d "'") +youtube-dl.tar.gz: README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish ChangeLog AUTHORS + @tar -czf youtube-dl-$(VERSION).tar.gz --transform "s|^|youtube-dl-$(VERSION)/|" --owner 0 --group 0 \ --exclude '*.DS_Store' \ --exclude '*.kate-swp' \ --exclude '*.pyc' \ @@ -132,4 +133,3 @@ youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash- ChangeLog AUTHORS LICENSE README.md README.txt \ Makefile MANIFEST.in youtube-dl.1 youtube-dl.bash-completion \ youtube-dl.zsh youtube-dl.fish setup.py setup.cfg \ - youtube-dl diff --git a/README.md b/README.md index 45326c69e..496bacf9c 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ -[![Build Status](https://travis-ci.org/ytdl-org/youtube-dl.svg?branch=master)](https://travis-ci.org/ytdl-org/youtube-dl) +[![Build Status](https://travis-ci.com/l1ving/youtube-dl.svg?branch=master)](https://travis-ci.com/l1ving/youtube-dl) youtube-dl - download videos from youtube.com or other video platforms +- [CHANGES](#changes) - [INSTALLATION](#installation) - [DESCRIPTION](#description) - [OPTIONS](#options) @@ -15,16 +16,24 @@ youtube-dl - download videos from youtube.com or other video platforms - [BUGS](#bugs) - [COPYRIGHT](#copyright) +# CHANGES + +You can view the changes made to ytdl-org/youtube-dl [here](https://github.com/l1ving/youtube-dl/compare/416da574ec0df3388f652e44f7fe71b1e3a4701f...master) + +You can view the archived tags here: [youtube-dl/releases](https://github.com/l1ving/youtube-dl/releases) + +You can view the archived unmerged pull requests here: [youtube-dl/tree/archive/recovered-github-prs](https://github.com/l1ving/youtube-dl/tree/archive/recovered-github-prs) + # INSTALLATION To install it right away for all UNIX users (Linux, macOS, etc.), type: - sudo curl -L https://yt-dl.org/downloads/latest/youtube-dl -o /usr/local/bin/youtube-dl + sudo curl -L https://github.com/l1ving/youtube-dl/releases/latest/download/youtube-dl -o /usr/local/bin/youtube-dl sudo chmod a+rx /usr/local/bin/youtube-dl If you do not have curl, you can alternatively use a recent wget: - sudo wget https://yt-dl.org/downloads/latest/youtube-dl -O /usr/local/bin/youtube-dl + sudo wget https://github.com/l1ving/youtube-dl/releases/latest/download/youtube-dl -O /usr/local/bin/youtube-dl sudo chmod a+rx /usr/local/bin/youtube-dl Windows users can [download an .exe file](https://yt-dl.org/latest/youtube-dl.exe) and place it in any location on their [PATH](https://en.wikipedia.org/wiki/PATH_%28variable%29) except for `%SYSTEMROOT%\System32` (e.g. **do not** put in `C:\Windows\System32`). @@ -545,7 +554,7 @@ The basic usage is not to set any template arguments when downloading a single f - `extractor` (string): Name of the extractor - `extractor_key` (string): Key name of the extractor - `epoch` (numeric): Unix epoch when creating the file - - `autonumber` (numeric): Five-digit number that will be increased with each download, starting at zero + - `autonumber` (numeric): Number that will be increased with each download, starting at `--autonumber-start` - `playlist` (string): Name or id of the playlist that contains the video - `playlist_index` (numeric): Index of the video in the playlist padded with leading zeros according to the total length of the playlist - `playlist_id` (string): Playlist identifier @@ -752,7 +761,7 @@ As a last resort, you can also uninstall the version installed by your package m Afterwards, simply follow [our manual installation instructions](https://ytdl-org.github.io/youtube-dl/download.html): ``` -sudo wget https://yt-dl.org/downloads/latest/youtube-dl -O /usr/local/bin/youtube-dl +sudo wget https://github.com/l1ving/youtube-dl/releases/latest/download/youtube-dl -O /usr/local/bin/youtube-dl sudo chmod a+rx /usr/local/bin/youtube-dl hash -r ``` diff --git a/test/test_all_urls.py b/test/test_all_urls.py index 81056a999..4f448b4f6 100644 --- a/test/test_all_urls.py +++ b/test/test_all_urls.py @@ -33,7 +33,6 @@ class TestAllURLsMatching(unittest.TestCase): assertPlaylist = lambda url: self.assertMatch(url, ['youtube:playlist']) assertPlaylist('ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8') assertPlaylist('UUBABnxM4Ar9ten8Mdjj1j0Q') # 585 - assertPlaylist('PL63F0C78739B09958') assertPlaylist('https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q') assertPlaylist('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8') assertPlaylist('https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC') diff --git a/test/test_utils.py b/test/test_utils.py index 962fd8d75..c2d1e4fb1 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -994,6 +994,12 @@ class TestUtil(unittest.TestCase): on = js_to_json('{42:4.2e1}') self.assertEqual(json.loads(on), {'42': 42.0}) + on = js_to_json('{ "0x40": "0x40" }') + self.assertEqual(json.loads(on), {'0x40': '0x40'}) + + on = js_to_json('{ "040": "040" }') + self.assertEqual(json.loads(on), {'040': '040'}) + def test_js_to_json_malformed(self): self.assertEqual(js_to_json('42a1'), '42"a1"') self.assertEqual(js_to_json('42a-1'), '42"a"-1') diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py index 84bc34928..0f2c06f40 100644 --- a/youtube_dl/downloader/hls.py +++ b/youtube_dl/downloader/hls.py @@ -141,7 +141,7 @@ class HlsFD(FragmentFD): count = 0 headers = info_dict.get('http_headers', {}) if byte_range: - headers['Range'] = 'bytes=%d-%d' % (byte_range['start'], byte_range['end']) + headers['Range'] = 'bytes=%d-%d' % (byte_range['start'], byte_range['end'] - 1) while count <= fragment_retries: try: success, frag_content = self._download_fragment( diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py index 5046878df..96379caf1 100644 --- a/youtube_dl/downloader/http.py +++ b/youtube_dl/downloader/http.py @@ -106,7 +106,12 @@ class HttpFD(FileDownloader): set_range(request, range_start, range_end) # Establish connection try: - ctx.data = self.ydl.urlopen(request) + try: + ctx.data = self.ydl.urlopen(request) + except (compat_urllib_error.URLError, ) as err: + if isinstance(err.reason, socket.timeout): + raise RetryDownload(err) + raise err # When trying to resume, Content-Range HTTP header of response has to be checked # to match the value of requested Range HTTP header. This is due to a webservers # that don't support resuming and serve a whole file with no Content-Range @@ -218,9 +223,10 @@ class HttpFD(FileDownloader): def retry(e): to_stdout = ctx.tmpfilename == '-' - if not to_stdout: - ctx.stream.close() - ctx.stream = None + if ctx.stream is not None: + if not to_stdout: + ctx.stream.close() + ctx.stream = None ctx.resume_len = byte_counter if to_stdout else os.path.getsize(encodeFilename(ctx.tmpfilename)) raise RetryDownload(e) @@ -233,9 +239,11 @@ class HttpFD(FileDownloader): except socket.timeout as e: retry(e) except socket.error as e: - if e.errno not in (errno.ECONNRESET, errno.ETIMEDOUT): - raise - retry(e) + # SSLError on python 2 (inherits socket.error) may have + # no errno set but this error message + if e.errno in (errno.ECONNRESET, errno.ETIMEDOUT) or getattr(e, 'message', None) == 'The read operation timed out': + retry(e) + raise byte_counter += len(data_block) diff --git a/youtube_dl/extractor/afreecatv.py b/youtube_dl/extractor/afreecatv.py index 6275e5209..b56abb1e6 100644 --- a/youtube_dl/extractor/afreecatv.py +++ b/youtube_dl/extractor/afreecatv.py @@ -275,7 +275,7 @@ class AfreecaTVIE(InfoExtractor): video_element = video_xml.findall(compat_xpath('./track/video'))[-1] if video_element is None or video_element.text is None: raise ExtractorError( - 'Video %s video does not exist' % video_id, expected=True) + 'Video %s does not exist' % video_id, expected=True) video_url = video_element.text.strip() diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index a61753b17..021945a89 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -10,6 +10,7 @@ import os import random import re import socket +import ssl import sys import time import math @@ -67,6 +68,7 @@ from ..utils import ( sanitized_Request, sanitize_filename, str_or_none, + str_to_int, strip_or_none, unescapeHTML, unified_strdate, @@ -623,9 +625,12 @@ class InfoExtractor(object): url_or_request = update_url_query(url_or_request, query) if data is not None or headers: url_or_request = sanitized_Request(url_or_request, data, headers) + exceptions = [compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error] + if hasattr(ssl, 'CertificateError'): + exceptions.append(ssl.CertificateError) try: return self._downloader.urlopen(url_or_request) - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + except tuple(exceptions) as err: if isinstance(err, compat_urllib_error.HTTPError): if self.__can_accept_status_code(err, expected_status): # Retain reference to error to prevent file object from @@ -1244,7 +1249,10 @@ class InfoExtractor(object): interaction_type = is_e.get('interactionType') if not isinstance(interaction_type, compat_str): continue - interaction_count = int_or_none(is_e.get('userInteractionCount')) + # For interaction count some sites provide string instead of + # an integer (as per spec) with non digit characters (e.g. ",") + # so extracting count with more relaxed str_to_int + interaction_count = str_to_int(is_e.get('userInteractionCount')) if interaction_count is None: continue count_kind = INTERACTION_TYPE_MAP.get(interaction_type.split('/')[-1]) @@ -1264,6 +1272,7 @@ class InfoExtractor(object): 'thumbnail': url_or_none(e.get('thumbnailUrl') or e.get('thumbnailURL')), 'duration': parse_duration(e.get('duration')), 'timestamp': unified_timestamp(e.get('uploadDate')), + 'uploader': str_or_none(e.get('author')), 'filesize': float_or_none(e.get('contentSize')), 'tbr': int_or_none(e.get('bitrate')), 'width': int_or_none(e.get('width')), diff --git a/youtube_dl/extractor/expressen.py b/youtube_dl/extractor/expressen.py index f79365038..dc8b855d2 100644 --- a/youtube_dl/extractor/expressen.py +++ b/youtube_dl/extractor/expressen.py @@ -15,7 +15,7 @@ from ..utils import ( class ExpressenIE(InfoExtractor): _VALID_URL = r'''(?x) https?:// - (?:www\.)?expressen\.se/ + (?:www\.)?(?:expressen|di)\.se/ (?:(?:tvspelare/video|videoplayer/embed)/)? tv/(?:[^/]+/)* (?P[^/?#&]+) @@ -42,13 +42,16 @@ class ExpressenIE(InfoExtractor): }, { 'url': 'https://www.expressen.se/videoplayer/embed/tv/ditv/ekonomistudion/experterna-har-ar-fragorna-som-avgor-valet/?embed=true&external=true&autoplay=true&startVolume=0&partnerId=di', 'only_matching': True, + }, { + 'url': 'https://www.di.se/videoplayer/embed/tv/ditv/borsmorgon/implantica-rusar-70--under-borspremiaren-hor-styrelsemedlemmen/?embed=true&external=true&autoplay=true&startVolume=0&partnerId=di', + 'only_matching': True, }] @staticmethod def _extract_urls(webpage): return [ mobj.group('url') for mobj in re.finditer( - r']+\bsrc=(["\'])(?P(?:https?:)?//(?:www\.)?expressen\.se/(?:tvspelare/video|videoplayer/embed)/tv/.+?)\1', + r']+\bsrc=(["\'])(?P(?:https?:)?//(?:www\.)?(?:expressen|di)\.se/(?:tvspelare/video|videoplayer/embed)/tv/.+?)\1', webpage)] def _real_extract(self, url): diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py index 53a550c11..648ae6741 100644 --- a/youtube_dl/extractor/iprima.py +++ b/youtube_dl/extractor/iprima.py @@ -86,7 +86,8 @@ class IPrimaIE(InfoExtractor): (r']+\bsrc=["\'](?:https?:)?//(?:api\.play-backend\.iprima\.cz/prehravac/embedded|prima\.iprima\.cz/[^/]+/[^/]+)\?.*?\bid=(p\d+)', r'data-product="([^"]+)">', r'id=["\']player-(p\d+)"', - r'playerId\s*:\s*["\']player-(p\d+)'), + r'playerId\s*:\s*["\']player-(p\d+)', + r'\bvideos\s*=\s*["\'](p\d+)'), webpage, 'real id') playerpage = self._download_webpage( diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py index cd11aa70f..5df674daf 100644 --- a/youtube_dl/extractor/iqiyi.py +++ b/youtube_dl/extractor/iqiyi.py @@ -150,7 +150,7 @@ class IqiyiSDKInterpreter(object): elif function in other_functions: other_functions[function]() else: - raise ExtractorError('Unknown funcion %s' % function) + raise ExtractorError('Unknown function %s' % function) return sdk.target diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py index 3567a3283..529f3f711 100644 --- a/youtube_dl/extractor/pornhub.py +++ b/youtube_dl/extractor/pornhub.py @@ -17,6 +17,7 @@ from ..utils import ( determine_ext, ExtractorError, int_or_none, + merge_dicts, NO_DEFAULT, orderedSet, remove_quotes, @@ -59,13 +60,14 @@ class PornHubIE(PornHubBaseIE): ''' _TESTS = [{ 'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015', - 'md5': '1e19b41231a02eba417839222ac9d58e', + 'md5': 'a6391306d050e4547f62b3f485dd9ba9', 'info_dict': { 'id': '648719015', 'ext': 'mp4', 'title': 'Seductive Indian beauty strips down and fingers her pink pussy', 'uploader': 'Babes', 'upload_date': '20130628', + 'timestamp': 1372447216, 'duration': 361, 'view_count': int, 'like_count': int, @@ -82,8 +84,8 @@ class PornHubIE(PornHubBaseIE): 'id': '1331683002', 'ext': 'mp4', 'title': '重庆婷婷女王足交', - 'uploader': 'Unknown', 'upload_date': '20150213', + 'timestamp': 1423804862, 'duration': 1753, 'view_count': int, 'like_count': int, @@ -121,6 +123,7 @@ class PornHubIE(PornHubBaseIE): 'params': { 'skip_download': True, }, + 'skip': 'This video has been disabled', }, { 'url': 'http://www.pornhub.com/view_video.php?viewkey=ph557bbb6676d2d', 'only_matching': True, @@ -338,10 +341,10 @@ class PornHubIE(PornHubBaseIE): video_uploader = self._html_search_regex( r'(?s)From: .+?<(?:a\b[^>]+\bhref=["\']/(?:(?:user|channel)s|model|pornstar)/|span\b[^>]+\bclass=["\']username)[^>]+>(.+?)<', - webpage, 'uploader', fatal=False) + webpage, 'uploader', default=None) view_count = self._extract_count( - r'([\d,\.]+) views', webpage, 'view') + r'([\d,\.]+) [Vv]iews', webpage, 'view') like_count = self._extract_count( r'([\d,\.]+)', webpage, 'like') dislike_count = self._extract_count( @@ -356,7 +359,11 @@ class PornHubIE(PornHubBaseIE): if div: return re.findall(r']+\bhref=[^>]+>([^<]+)', div) - return { + info = self._search_json_ld(webpage, video_id, default={}) + # description provided in JSON-LD is irrelevant + info['description'] = None + + return merge_dicts({ 'id': video_id, 'uploader': video_uploader, 'upload_date': upload_date, @@ -372,7 +379,7 @@ class PornHubIE(PornHubBaseIE): 'tags': extract_list('tags'), 'categories': extract_list('categories'), 'subtitles': subtitles, - } + }, info) class PornHubPlaylistBaseIE(PornHubBaseIE): diff --git a/youtube_dl/extractor/redtube.py b/youtube_dl/extractor/redtube.py index 2d2f6a98c..a1ca791ca 100644 --- a/youtube_dl/extractor/redtube.py +++ b/youtube_dl/extractor/redtube.py @@ -15,7 +15,7 @@ from ..utils import ( class RedTubeIE(InfoExtractor): - _VALID_URL = r'https?://(?:(?:www\.)?redtube\.com/|embed\.redtube\.com/\?.*?\bid=)(?P[0-9]+)' + _VALID_URL = r'https?://(?:(?:\w+\.)?redtube\.com/|embed\.redtube\.com/\?.*?\bid=)(?P[0-9]+)' _TESTS = [{ 'url': 'http://www.redtube.com/66418', 'md5': 'fc08071233725f26b8f014dba9590005', @@ -31,6 +31,9 @@ class RedTubeIE(InfoExtractor): }, { 'url': 'http://embed.redtube.com/?bgcolor=000000&id=1443286', 'only_matching': True, + }, { + 'url': 'http://it.redtube.com/66418', + 'only_matching': True, }] @staticmethod diff --git a/youtube_dl/extractor/telequebec.py b/youtube_dl/extractor/telequebec.py index c82c94b3a..b4c485b9b 100644 --- a/youtube_dl/extractor/telequebec.py +++ b/youtube_dl/extractor/telequebec.py @@ -13,14 +13,24 @@ from ..utils import ( class TeleQuebecBaseIE(InfoExtractor): @staticmethod - def _limelight_result(media_id): + def _result(url, ie_key): return { '_type': 'url_transparent', - 'url': smuggle_url( - 'limelight:media:' + media_id, {'geo_countries': ['CA']}), - 'ie_key': 'LimelightMedia', + 'url': smuggle_url(url, {'geo_countries': ['CA']}), + 'ie_key': ie_key, } + @staticmethod + def _limelight_result(media_id): + return TeleQuebecBaseIE._result( + 'limelight:media:' + media_id, 'LimelightMedia') + + @staticmethod + def _brightcove_result(brightcove_id): + return TeleQuebecBaseIE._result( + 'http://players.brightcove.net/6150020952001/default_default/index.html?videoId=%s' + % brightcove_id, 'BrightcoveNew') + class TeleQuebecIE(TeleQuebecBaseIE): _VALID_URL = r'''(?x) @@ -37,11 +47,27 @@ class TeleQuebecIE(TeleQuebecBaseIE): 'id': '577116881b4b439084e6b1cf4ef8b1b3', 'ext': 'mp4', 'title': 'Un petit choc et puis repart!', - 'description': 'md5:b04a7e6b3f74e32d7b294cffe8658374', + 'description': 'md5:067bc84bd6afecad85e69d1000730907', }, 'params': { 'skip_download': True, }, + }, { + 'url': 'https://zonevideo.telequebec.tv/media/55267/le-soleil/passe-partout', + 'info_dict': { + 'id': '6167180337001', + 'ext': 'mp4', + 'title': 'Le soleil', + 'description': 'md5:64289c922a8de2abbe99c354daffde02', + 'uploader_id': '6150020952001', + 'upload_date': '20200625', + 'timestamp': 1593090307, + }, + 'params': { + 'format': 'bestvideo', + 'skip_download': True, + }, + 'add_ie': ['BrightcoveNew'], }, { # no description 'url': 'http://zonevideo.telequebec.tv/media/30261', @@ -58,7 +84,14 @@ class TeleQuebecIE(TeleQuebecBaseIE): 'https://mnmedias.api.telequebec.tv/api/v2/media/' + media_id, media_id)['media'] - info = self._limelight_result(media_data['streamInfo']['sourceId']) + source_id = media_data['streamInfo']['sourceId'] + source = (try_get( + media_data, lambda x: x['streamInfo']['source'], + compat_str) or 'limelight').lower() + if source == 'brightcove': + info = self._brightcove_result(source_id) + else: + info = self._limelight_result(source_id) info.update({ 'title': media_data.get('title'), 'description': try_get( diff --git a/youtube_dl/extractor/twentythreevideo.py b/youtube_dl/extractor/twentythreevideo.py index aa0c6e90f..dc5609192 100644 --- a/youtube_dl/extractor/twentythreevideo.py +++ b/youtube_dl/extractor/twentythreevideo.py @@ -8,8 +8,8 @@ from ..utils import int_or_none class TwentyThreeVideoIE(InfoExtractor): IE_NAME = '23video' - _VALID_URL = r'https?://video\.(?Ptwentythree\.net|23video\.com|filmweb\.no)/v\.ihtml/player\.html\?(?P.*?\bphoto(?:_|%5f)id=(?P\d+).*)' - _TEST = { + _VALID_URL = r'https?://(?P[^.]+\.(?:twentythree\.net|23video\.com|filmweb\.no))/v\.ihtml/player\.html\?(?P.*?\bphoto(?:_|%5f)id=(?P\d+).*)' + _TESTS = [{ 'url': 'https://video.twentythree.net/v.ihtml/player.html?showDescriptions=0&source=site&photo%5fid=20448876&autoPlay=1', 'md5': '75fcf216303eb1dae9920d651f85ced4', 'info_dict': { @@ -21,11 +21,14 @@ class TwentyThreeVideoIE(InfoExtractor): 'uploader_id': '12258964', 'uploader': 'Rasmus Bysted', } - } + }, { + 'url': 'https://bonnier-publications-danmark.23video.com/v.ihtml/player.html?token=f0dc46476e06e13afd5a1f84a29e31e8&source=embed&photo%5fid=36137620', + 'only_matching': True, + }] def _real_extract(self, url): domain, query, photo_id = re.match(self._VALID_URL, url).groups() - base_url = 'https://video.%s' % domain + base_url = 'https://%s' % domain photo_data = self._download_json( base_url + '/api/photo/list?' + query, photo_id, query={ 'format': 'json', diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py index eadc48c6d..ab6654432 100644 --- a/youtube_dl/extractor/twitch.py +++ b/youtube_dl/extractor/twitch.py @@ -24,7 +24,6 @@ from ..utils import ( parse_duration, parse_iso8601, qualities, - str_or_none, try_get, unified_timestamp, update_url_query, @@ -337,19 +336,27 @@ def _make_video_result(node): class TwitchGraphQLBaseIE(TwitchBaseIE): _PAGE_LIMIT = 100 - def _download_gql(self, video_id, op, variables, sha256_hash, note, fatal=True): + _OPERATION_HASHES = { + 'CollectionSideBar': '27111f1b382effad0b6def325caef1909c733fe6a4fbabf54f8d491ef2cf2f14', + 'FilterableVideoTower_Videos': 'a937f1d22e269e39a03b509f65a7490f9fc247d7f83d6ac1421523e3b68042cb', + 'ClipsCards__User': 'b73ad2bfaecfd30a9e6c28fada15bd97032c83ec77a0440766a56fe0bd632777', + 'ChannelCollectionsContent': '07e3691a1bad77a36aba590c351180439a40baefc1c275356f40fc7082419a84', + 'StreamMetadata': '1c719a40e481453e5c48d9bb585d971b8b372f8ebb105b17076722264dfa5b3e', + 'ComscoreStreamingQuery': 'e1edae8122517d013405f237ffcc124515dc6ded82480a88daef69c83b53ac01', + 'VideoPreviewOverlay': '3006e77e51b128d838fa4e835723ca4dc9a05c5efd4466c1085215c6e437e65c', + } + + def _download_gql(self, video_id, ops, note, fatal=True): + for op in ops: + op['extensions'] = { + 'persistedQuery': { + 'version': 1, + 'sha256Hash': self._OPERATION_HASHES[op['operationName']], + } + } return self._download_json( 'https://gql.twitch.tv/gql', video_id, note, - data=json.dumps({ - 'operationName': op, - 'variables': variables, - 'extensions': { - 'persistedQuery': { - 'version': 1, - 'sha256Hash': sha256_hash, - } - } - }).encode(), + data=json.dumps(ops).encode(), headers={ 'Content-Type': 'text/plain;charset=UTF-8', 'Client-ID': self._CLIENT_ID, @@ -369,14 +376,15 @@ class TwitchCollectionIE(TwitchGraphQLBaseIE): }] _OPERATION_NAME = 'CollectionSideBar' - _SHA256_HASH = '27111f1b382effad0b6def325caef1909c733fe6a4fbabf54f8d491ef2cf2f14' def _real_extract(self, url): collection_id = self._match_id(url) collection = self._download_gql( - collection_id, self._OPERATION_NAME, - {'collectionID': collection_id}, self._SHA256_HASH, - 'Downloading collection GraphQL')['data']['collection'] + collection_id, [{ + 'operationName': self._OPERATION_NAME, + 'variables': {'collectionID': collection_id}, + }], + 'Downloading collection GraphQL')[0]['data']['collection'] title = collection.get('title') entries = [] for edge in collection['items']['edges']: @@ -403,14 +411,16 @@ class TwitchPlaylistBaseIE(TwitchGraphQLBaseIE): if cursor: variables['cursor'] = cursor page = self._download_gql( - channel_name, self._OPERATION_NAME, variables, - self._SHA256_HASH, + channel_name, [{ + 'operationName': self._OPERATION_NAME, + 'variables': variables, + }], 'Downloading %ss GraphQL page %s' % (self._NODE_KIND, page_num), fatal=False) if not page: break edges = try_get( - page, lambda x: x['data']['user'][entries_key]['edges'], list) + page, lambda x: x[0]['data']['user'][entries_key]['edges'], list) if not edges: break for edge in edges: @@ -553,7 +563,6 @@ class TwitchVideosIE(TwitchPlaylistBaseIE): 'views': 'Popular', } - _SHA256_HASH = 'a937f1d22e269e39a03b509f65a7490f9fc247d7f83d6ac1421523e3b68042cb' _OPERATION_NAME = 'FilterableVideoTower_Videos' _ENTRY_KIND = 'video' _EDGE_KIND = 'VideoEdge' @@ -622,7 +631,6 @@ class TwitchVideosClipsIE(TwitchPlaylistBaseIE): # NB: values other than 20 result in skipped videos _PAGE_LIMIT = 20 - _SHA256_HASH = 'b73ad2bfaecfd30a9e6c28fada15bd97032c83ec77a0440766a56fe0bd632777' _OPERATION_NAME = 'ClipsCards__User' _ENTRY_KIND = 'clip' _EDGE_KIND = 'ClipEdge' @@ -680,7 +688,6 @@ class TwitchVideosCollectionsIE(TwitchPlaylistBaseIE): 'playlist_mincount': 3, }] - _SHA256_HASH = '07e3691a1bad77a36aba590c351180439a40baefc1c275356f40fc7082419a84' _OPERATION_NAME = 'ChannelCollectionsContent' _ENTRY_KIND = 'collection' _EDGE_KIND = 'CollectionsItemEdge' @@ -717,7 +724,7 @@ class TwitchVideosCollectionsIE(TwitchPlaylistBaseIE): playlist_title='%s - Collections' % channel_name) -class TwitchStreamIE(TwitchBaseIE): +class TwitchStreamIE(TwitchGraphQLBaseIE): IE_NAME = 'twitch:stream' _VALID_URL = r'''(?x) https?:// @@ -774,28 +781,43 @@ class TwitchStreamIE(TwitchBaseIE): else super(TwitchStreamIE, cls).suitable(url)) def _real_extract(self, url): - channel_name = self._match_id(url) + channel_name = self._match_id(url).lower() - access_token = self._download_access_token(channel_name) + gql = self._download_gql( + channel_name, [{ + 'operationName': 'StreamMetadata', + 'variables': {'channelLogin': channel_name}, + }, { + 'operationName': 'ComscoreStreamingQuery', + 'variables': { + 'channel': channel_name, + 'clipSlug': '', + 'isClip': False, + 'isLive': True, + 'isVodOrCollection': False, + 'vodID': '', + }, + }, { + 'operationName': 'VideoPreviewOverlay', + 'variables': {'login': channel_name}, + }], + 'Downloading stream GraphQL') - token = access_token['token'] - channel_id = self._extract_channel_id(token, channel_name) + user = gql[0]['data']['user'] - stream = self._call_api( - 'kraken/streams/%s?stream_type=all' % channel_id, - channel_id, 'Downloading stream JSON').get('stream') + if not user: + raise ExtractorError( + '%s does not exist' % channel_name, expected=True) + + stream = user['stream'] if not stream: - raise ExtractorError('%s is offline' % channel_id, expected=True) + raise ExtractorError('%s is offline' % channel_name, expected=True) - # Channel name may be typed if different case than the original channel name - # (e.g. http://www.twitch.tv/TWITCHPLAYSPOKEMON) that will lead to constructing - # an invalid m3u8 URL. Working around by use of original channel name from stream - # JSON and fallback to lowercase if it's not available. - channel_name = try_get( - stream, lambda x: x['channel']['name'], - compat_str) or channel_name.lower() + access_token = self._download_access_token(channel_name) + token = access_token['token'] + stream_id = stream.get('id') or channel_name query = { 'allow_source': 'true', 'allow_audio_only': 'true', @@ -808,41 +830,39 @@ class TwitchStreamIE(TwitchBaseIE): 'token': token.encode('utf-8'), } formats = self._extract_m3u8_formats( - '%s/api/channel/hls/%s.m3u8?%s' - % (self._USHER_BASE, channel_name, compat_urllib_parse_urlencode(query)), - channel_id, 'mp4') + '%s/api/channel/hls/%s.m3u8' % (self._USHER_BASE, channel_name), + stream_id, 'mp4', query=query) self._prefer_source(formats) view_count = stream.get('viewers') - timestamp = parse_iso8601(stream.get('created_at')) + timestamp = unified_timestamp(stream.get('createdAt')) - channel = stream['channel'] - title = self._live_title(channel.get('display_name') or channel.get('name')) - description = channel.get('status') + sq_user = try_get(gql, lambda x: x[1]['data']['user'], dict) or {} + uploader = sq_user.get('displayName') + description = try_get( + sq_user, lambda x: x['broadcastSettings']['title'], compat_str) - thumbnails = [] - for thumbnail_key, thumbnail_url in stream['preview'].items(): - m = re.search(r'(?P\d+)x(?P\d+)\.jpg$', thumbnail_key) - if not m: - continue - thumbnails.append({ - 'url': thumbnail_url, - 'width': int(m.group('width')), - 'height': int(m.group('height')), - }) + thumbnail = url_or_none(try_get( + gql, lambda x: x[2]['data']['user']['stream']['previewImageURL'], + compat_str)) + + title = uploader or channel_name + stream_type = stream.get('type') + if stream_type in ['rerun', 'live']: + title += ' (%s)' % stream_type return { - 'id': str_or_none(stream.get('_id')) or channel_id, + 'id': stream_id, 'display_id': channel_name, - 'title': title, + 'title': self._live_title(title), 'description': description, - 'thumbnails': thumbnails, - 'uploader': channel.get('display_name'), - 'uploader_id': channel.get('name'), + 'thumbnail': thumbnail, + 'uploader': uploader, + 'uploader_id': channel_name, 'timestamp': timestamp, 'view_count': view_count, 'formats': formats, - 'is_live': True, + 'is_live': stream_type == 'live', } diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py index 582090d0d..9e860aeb7 100644 --- a/youtube_dl/extractor/ustream.py +++ b/youtube_dl/extractor/ustream.py @@ -19,7 +19,7 @@ from ..utils import ( class UstreamIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?ustream\.tv/(?Precorded|embed|embed/recorded)/(?P\d+)' + _VALID_URL = r'https?://(?:www\.)?(?:ustream\.tv|video\.ibm\.com)/(?Precorded|embed|embed/recorded)/(?P\d+)' IE_NAME = 'ustream' _TESTS = [{ 'url': 'http://www.ustream.tv/recorded/20274954', @@ -67,12 +67,15 @@ class UstreamIE(InfoExtractor): 'params': { 'skip_download': True, # m3u8 download }, + }, { + 'url': 'https://video.ibm.com/embed/recorded/128240221?&autoplay=true&controls=true&volume=100', + 'only_matching': True, }] @staticmethod def _extract_url(webpage): mobj = re.search( - r']+?src=(["\'])(?Phttp://www\.ustream\.tv/embed/.+?)\1', webpage) + r']+?src=(["\'])(?Phttp://(?:www\.)?(?:ustream\.tv|video\.ibm\.com)/embed/.+?)\1', webpage) if mobj is not None: return mobj.group('url') diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 02f3ab61a..e6ca51dee 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -579,44 +579,18 @@ class YoutubeIE(YoutubeBaseInfoExtractor): } }, { - 'url': 'https://www.youtube.com/watch?v=UxxajLWwzqY', - 'note': 'Test generic use_cipher_signature video (#897)', + 'url': 'https://www.youtube.com/watch?v=PA4gbtKWNAI', + 'note': 'Test video with age protection', 'info_dict': { - 'id': 'UxxajLWwzqY', + 'id': 'PA4gbtKWNAI', 'ext': 'mp4', - 'upload_date': '20120506', - 'title': 'Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]', - 'alt_title': 'I Love It (feat. Charli XCX)', - 'description': 'md5:19a2f98d9032b9311e686ed039564f63', - 'tags': ['Icona Pop i love it', 'sweden', 'pop music', 'big beat records', 'big beat', 'charli', - 'xcx', 'charli xcx', 'girls', 'hbo', 'i love it', "i don't care", 'icona', 'pop', - 'iconic ep', 'iconic', 'love', 'it'], - 'duration': 180, - 'uploader': 'Icona Pop', - 'uploader_id': 'IconaPop', - 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IconaPop', - 'creator': 'Icona Pop', - 'track': 'I Love It (feat. Charli XCX)', - 'artist': 'Icona Pop', - } - }, - { - 'url': 'https://www.youtube.com/watch?v=07FYdnEawAQ', - 'note': 'Test VEVO video with age protection (#956)', - 'info_dict': { - 'id': '07FYdnEawAQ', - 'ext': 'mp4', - 'upload_date': '20130703', - 'title': 'Justin Timberlake - Tunnel Vision (Official Music Video) (Explicit)', - 'alt_title': 'Tunnel Vision', - 'description': 'md5:07dab3356cde4199048e4c7cd93471e1', - 'duration': 419, - 'uploader': 'justintimberlakeVEVO', - 'uploader_id': 'justintimberlakeVEVO', - 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/justintimberlakeVEVO', - 'creator': 'Justin Timberlake', - 'track': 'Tunnel Vision', - 'artist': 'Justin Timberlake', + 'upload_date': '20201028', + 'title': 'Big Buck Bunny (TEST) (Age-restricted)', + 'description': 'An age restricted test video for youtube-dl', + 'duration': 635, + 'uploader': 'Ali Sherief', + 'uploader_id': 'UCi1TsEkfcMaYSadGms3UnqA', + 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCi1TsEkfcMaYSadGms3UnqA', 'age_limit': 18, } }, @@ -677,42 +651,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor): }, 'skip': 'format 141 not served anymore', }, - # DASH manifest with encrypted signature - { - 'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA', - 'info_dict': { - 'id': 'IB3lcPjvWLA', - 'ext': 'm4a', - 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson', - 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf', - 'duration': 244, - 'uploader': 'AfrojackVEVO', - 'uploader_id': 'AfrojackVEVO', - 'upload_date': '20131011', - }, - 'params': { - 'youtube_include_dash_manifest': True, - 'format': '141/bestaudio[ext=m4a]', - }, - }, - # JS player signature function name containing $ - { - 'url': 'https://www.youtube.com/watch?v=nfWlot6h_JM', - 'info_dict': { - 'id': 'nfWlot6h_JM', - 'ext': 'm4a', - 'title': 'Taylor Swift - Shake It Off', - 'description': 'md5:307195cd21ff7fa352270fe884570ef0', - 'duration': 242, - 'uploader': 'TaylorSwiftVEVO', - 'uploader_id': 'TaylorSwiftVEVO', - 'upload_date': '20140818', - }, - 'params': { - 'youtube_include_dash_manifest': True, - 'format': '141/bestaudio[ext=m4a]', - }, - }, # Controversy video { 'url': 'https://www.youtube.com/watch?v=T4XJQO3qol8', @@ -728,59 +666,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor): 'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms\n\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html', } }, - # Normal age-gate video (No vevo, embed allowed) - { - 'url': 'https://youtube.com/watch?v=HtVdAasjOgU', - 'info_dict': { - 'id': 'HtVdAasjOgU', - 'ext': 'mp4', - 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer', - 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}', - 'duration': 142, - 'uploader': 'The Witcher', - 'uploader_id': 'WitcherGame', - 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame', - 'upload_date': '20140605', - 'age_limit': 18, - }, - }, - # Age-gate video with encrypted signature - { - 'url': 'https://www.youtube.com/watch?v=6kLq3WMV1nU', - 'info_dict': { - 'id': '6kLq3WMV1nU', - 'ext': 'mp4', - 'title': 'Dedication To My Ex (Miss That) (Lyric Video)', - 'description': 'md5:33765bb339e1b47e7e72b5490139bb41', - 'duration': 246, - 'uploader': 'LloydVEVO', - 'uploader_id': 'LloydVEVO', - 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/LloydVEVO', - 'upload_date': '20110629', - 'age_limit': 18, - }, - }, - # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421) - # YouTube Red ad is not captured for creator - { - 'url': '__2ABJjxzNo', - 'info_dict': { - 'id': '__2ABJjxzNo', - 'ext': 'mp4', - 'duration': 266, - 'upload_date': '20100430', - 'uploader_id': 'deadmau5', - 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5', - 'creator': 'Dada Life, deadmau5', - 'description': 'md5:12c56784b8032162bb936a5f76d55360', - 'uploader': 'deadmau5', - 'title': 'Deadmau5 - Some Chords (HD)', - 'alt_title': 'This Machine Kills Some Chords', - }, - 'expected_warnings': [ - 'DASH manifest missing', - ] - }, # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431) { 'url': 'lqQg6PlCWgI', @@ -2086,7 +1971,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor): if cipher: if 's' in url_data or self._downloader.params.get('youtube_include_dash_manifest', True): - ASSETS_RE = r'"assets":.+?"js":\s*("[^"]+")' + ASSETS_RE = ( + r']+\bsrc=("[^"]+")[^>]+\bname=["\']player_ias/base', + r'"jsUrl"\s*:\s*("[^"]+")', + r'"assets":.+?"js":\s*("[^"]+")') jsplayer_url_json = self._search_regex( ASSETS_RE, embed_webpage if age_gate else video_webpage, @@ -3181,54 +3069,94 @@ class YoutubeSearchIE(SearchInfoExtractor, YoutubeSearchBaseInfoExtractor): _MAX_RESULTS = float('inf') IE_NAME = 'youtube:search' _SEARCH_KEY = 'ytsearch' - _EXTRA_QUERY_ARGS = {} + _SEARCH_PARAMS = None _TESTS = [] + def _entries(self, query, n): + data = { + 'context': { + 'client': { + 'clientName': 'WEB', + 'clientVersion': '2.20201021.03.00', + } + }, + 'query': query, + } + if self._SEARCH_PARAMS: + data['params'] = self._SEARCH_PARAMS + total = 0 + for page_num in itertools.count(1): + search = self._download_json( + 'https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8', + video_id='query "%s"' % query, + note='Downloading page %s' % page_num, + errnote='Unable to download API page', fatal=False, + data=json.dumps(data).encode('utf8'), + headers={'content-type': 'application/json'}) + if not search: + break + slr_contents = try_get( + search, + (lambda x: x['contents']['twoColumnSearchResultsRenderer']['primaryContents']['sectionListRenderer']['contents'], + lambda x: x['onResponseReceivedCommands'][0]['appendContinuationItemsAction']['continuationItems']), + list) + if not slr_contents: + break + isr_contents = try_get( + slr_contents, + lambda x: x[0]['itemSectionRenderer']['contents'], + list) + if not isr_contents: + break + for content in isr_contents: + if not isinstance(content, dict): + continue + video = content.get('videoRenderer') + if not isinstance(video, dict): + continue + video_id = video.get('videoId') + if not video_id: + continue + title = try_get(video, lambda x: x['title']['runs'][0]['text'], compat_str) + description = try_get(video, lambda x: x['descriptionSnippet']['runs'][0]['text'], compat_str) + duration = parse_duration(try_get(video, lambda x: x['lengthText']['simpleText'], compat_str)) + view_count_text = try_get(video, lambda x: x['viewCountText']['simpleText'], compat_str) or '' + view_count = int_or_none(self._search_regex( + r'^(\d+)', re.sub(r'\s', '', view_count_text), + 'view count', default=None)) + uploader = try_get(video, lambda x: x['ownerText']['runs'][0]['text'], compat_str) + total += 1 + yield { + '_type': 'url_transparent', + 'ie_key': YoutubeIE.ie_key(), + 'id': video_id, + 'url': video_id, + 'title': title, + 'description': description, + 'duration': duration, + 'view_count': view_count, + 'uploader': uploader, + } + if total == n: + return + token = try_get( + slr_contents, + lambda x: x[1]['continuationItemRenderer']['continuationEndpoint']['continuationCommand']['token'], + compat_str) + if not token: + break + data['continuation'] = token + def _get_n_results(self, query, n): """Get a specified number of results for a query""" - - videos = [] - limit = n - - url_query = { - 'search_query': query.encode('utf-8'), - } - url_query.update(self._EXTRA_QUERY_ARGS) - result_url = 'https://www.youtube.com/results?' + compat_urllib_parse_urlencode(url_query) - - for pagenum in itertools.count(1): - data = self._download_json( - result_url, video_id='query "%s"' % query, - note='Downloading page %s' % pagenum, - errnote='Unable to download API page', - query={'spf': 'navigate'}) - html_content = data[1]['body']['content'] - - if 'class="search-message' in html_content: - raise ExtractorError( - '[youtube] No video results', expected=True) - - new_videos = list(self._process_page(html_content)) - videos += new_videos - if not new_videos or len(videos) > limit: - break - next_link = self._html_search_regex( - r'href="(/results\?[^"]*\bsp=[^"]+)"[^>]*>\s*]+class="[^"]*\byt-uix-button-content\b[^"]*"[^>]*>Next', - html_content, 'next link', default=None) - if next_link is None: - break - result_url = compat_urlparse.urljoin('https://www.youtube.com/', next_link) - - if len(videos) > n: - videos = videos[:n] - return self.playlist_result(videos, query) + return self.playlist_result(self._entries(query, n), query) class YoutubeSearchDateIE(YoutubeSearchIE): IE_NAME = YoutubeSearchIE.IE_NAME + ':date' _SEARCH_KEY = 'ytsearchdate' IE_DESC = 'YouTube.com searches, newest videos first' - _EXTRA_QUERY_ARGS = {'search_sort': 'video_date_uploaded'} + _SEARCH_PARAMS = 'CAI%3D' class YoutubeSearchURLIE(YoutubeSearchBaseInfoExtractor): diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 01d9c0362..737e2810e 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -4088,12 +4088,12 @@ def js_to_json(code): '\\\n': '', '\\x': '\\u00', }.get(m.group(0), m.group(0)), v[1:-1]) - - for regex, base in INTEGER_TABLE: - im = re.match(regex, v) - if im: - i = int(im.group(1), base) - return '"%d":' % i if v.endswith(':') else '%d' % i + else: + for regex, base in INTEGER_TABLE: + im = re.match(regex, v) + if im: + i = int(im.group(1), base) + return '"%d":' % i if v.endswith(':') else '%d' % i return '"%s"' % v diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 5625b8324..7d89dd3e1 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '2020.09.14' +__version__ = '2020.11.01'