diff options
| author | Tithen-Firion <Tithen-Firion@users.noreply.github.com> | 2014-12-04 14:11:27 +0100 |
|---|---|---|
| committer | Tithen-Firion <Tithen-Firion@users.noreply.github.com> | 2014-12-04 14:16:09 +0100 |
| commit | 995ad69c54899a0cfc84fd89083f07919acdbb83 (patch) | |
| tree | 91992f4c4b3b406ea82366c7108de460a922e900 | |
| parent | 225e4b9633285b66adc914a61d8f55ca125eb91d (diff) | |
| download | youtube-dl-995ad69c54899a0cfc84fd89083f07919acdbb83.tar.gz youtube-dl-995ad69c54899a0cfc84fd89083f07919acdbb83.tar.xz youtube-dl-995ad69c54899a0cfc84fd89083f07919acdbb83.zip | |
[common] Add new parameters for _download_webpage
| -rw-r--r-- | youtube_dl/extractor/common.py | 14 |
1 files changed, 12 insertions, 2 deletions
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index e80a2dad0..b633ea9b9 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -360,9 +360,19 @@ class InfoExtractor(object): return content - def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True): + def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5): """ Returns the data of the page as a string """ - res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal) + success = False + try_count = 0 + while success is False: + try: + res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal) + success = True + except compat_http_client.IncompleteRead as e: + try_count += 1 + if try_count >= tries: + raise e + self._sleep(timeout, video_id) if res is False: return res else: |