From 7097bffba6769302d9aef7e839baf4c6f30de066 Mon Sep 17 00:00:00 2001
From: Sergey M․ <dstftw@gmail.com>
Date: Mon, 6 Feb 2017 23:07:59 +0700
Subject: [downloader/fragment] Respect --no-part

---
 youtube_dl/downloader/fragment.py | 1 +
 1 file changed, 1 insertion(+)

(limited to 'youtube_dl/downloader/fragment.py')

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 60df627a6..56f975266 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -61,6 +61,7 @@ class FragmentFD(FileDownloader):
                 'noprogress': True,
                 'ratelimit': self.params.get('ratelimit'),
                 'retries': self.params.get('retries', 0),
+                'nopart': self.params.get('nopart', False),
                 'test': self.params.get('test', False),
             }
         )
-- 
cgit 1.4.1


From 75a24854073e590f4efc9f037b57dee348f52b61 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 28 Jun 2016 18:07:50 +0100
Subject: [fragment,hls,f4m,dash,ism] improve fragment downloading

- resume immediately
- no need to concatenate segments and decrypt them on every resume
- no need to save temp files for segments

and for hls downloader:
- no need to download keys for segments that already downloaded
---
 youtube_dl/downloader/common.py   | 31 ++++++++++-----------
 youtube_dl/downloader/dash.py     | 43 ++++++++---------------------
 youtube_dl/downloader/f4m.py      | 33 +++++++----------------
 youtube_dl/downloader/fragment.py | 57 ++++++++++++++++++++++++++++++++-------
 youtube_dl/downloader/hls.py      | 34 +++++++++--------------
 youtube_dl/downloader/http.py     |  8 ++++--
 youtube_dl/downloader/ism.py      | 30 +++++++--------------
 7 files changed, 112 insertions(+), 124 deletions(-)

(limited to 'youtube_dl/downloader/fragment.py')

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 2c4470a95..fdb77b620 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -327,21 +327,22 @@ class FileDownloader(object):
             os.path.exists(encodeFilename(filename))
         )
 
-        continuedl_and_exists = (
-            self.params.get('continuedl', True) and
-            os.path.isfile(encodeFilename(filename)) and
-            not self.params.get('nopart', False)
-        )
-
-        # Check file already present
-        if filename != '-' and (nooverwrites_and_exists or continuedl_and_exists):
-            self.report_file_already_downloaded(filename)
-            self._hook_progress({
-                'filename': filename,
-                'status': 'finished',
-                'total_bytes': os.path.getsize(encodeFilename(filename)),
-            })
-            return True
+        if not hasattr(filename, 'write'):
+            continuedl_and_exists = (
+                self.params.get('continuedl', True) and
+                os.path.isfile(encodeFilename(filename)) and
+                not self.params.get('nopart', False)
+            )
+
+            # Check file already present
+            if filename != '-' and (nooverwrites_and_exists or continuedl_and_exists):
+                self.report_file_already_downloaded(filename)
+                self._hook_progress({
+                    'filename': filename,
+                    'status': 'finished',
+                    'total_bytes': os.path.getsize(encodeFilename(filename)),
+                })
+                return True
 
         min_sleep_interval = self.params.get('sleep_interval')
         if min_sleep_interval:
diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index e2ddc369e..94a13a543 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -1,13 +1,7 @@
 from __future__ import unicode_literals
 
-import os
-
 from .fragment import FragmentFD
 from ..compat import compat_urllib_error
-from ..utils import (
-    sanitize_open,
-    encodeFilename,
-)
 
 
 class DashSegmentsFD(FragmentFD):
@@ -28,31 +22,24 @@ class DashSegmentsFD(FragmentFD):
 
         self._prepare_and_start_frag_download(ctx)
 
-        segments_filenames = []
-
         fragment_retries = self.params.get('fragment_retries', 0)
         skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
 
-        def process_segment(segment, tmp_filename, num):
-            segment_url = segment['url']
-            segment_name = 'Frag%d' % num
-            target_filename = '%s-%s' % (tmp_filename, segment_name)
+        frag_index = 0
+        for i, segment in enumerate(segments):
+            frag_index += 1
+            if frag_index <= ctx['frag_index']:
+                continue
             # In DASH, the first segment contains necessary headers to
             # generate a valid MP4 file, so always abort for the first segment
-            fatal = num == 0 or not skip_unavailable_fragments
+            fatal = i == 0 or not skip_unavailable_fragments
             count = 0
             while count <= fragment_retries:
                 try:
-                    success = ctx['dl'].download(target_filename, {
-                        'url': segment_url,
-                        'http_headers': info_dict.get('http_headers'),
-                    })
+                    success, frag_content = self._download_fragment(ctx, segment['url'], info_dict)
                     if not success:
                         return False
-                    down, target_sanitized = sanitize_open(target_filename, 'rb')
-                    ctx['dest_stream'].write(down.read())
-                    down.close()
-                    segments_filenames.append(target_sanitized)
+                    self._append_fragment(ctx, frag_content)
                     break
                 except compat_urllib_error.HTTPError as err:
                     # YouTube may often return 404 HTTP error for a fragment causing the
@@ -63,22 +50,14 @@ class DashSegmentsFD(FragmentFD):
                     # HTTP error.
                     count += 1
                     if count <= fragment_retries:
-                        self.report_retry_fragment(err, segment_name, count, fragment_retries)
+                        self.report_retry_fragment(err, frag_index, count, fragment_retries)
             if count > fragment_retries:
                 if not fatal:
-                    self.report_skip_fragment(segment_name)
-                    return True
+                    self.report_skip_fragment(frag_index)
+                    continue
                 self.report_error('giving up after %s fragment retries' % fragment_retries)
                 return False
-            return True
-
-        for i, segment in enumerate(segments):
-            if not process_segment(segment, ctx['tmpfilename'], i):
-                return False
 
         self._finish_frag_download(ctx)
 
-        for segment_file in segments_filenames:
-            os.remove(encodeFilename(segment_file))
-
         return True
diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 688e086eb..e456ed58f 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -3,7 +3,6 @@ from __future__ import division, unicode_literals
 import base64
 import io
 import itertools
-import os
 import time
 
 from .fragment import FragmentFD
@@ -16,9 +15,7 @@ from ..compat import (
     compat_struct_unpack,
 )
 from ..utils import (
-    encodeFilename,
     fix_xml_ampersands,
-    sanitize_open,
     xpath_text,
 )
 
@@ -366,17 +363,21 @@ class F4mFD(FragmentFD):
 
         dest_stream = ctx['dest_stream']
 
-        write_flv_header(dest_stream)
-        if not live:
-            write_metadata_tag(dest_stream, metadata)
+        if ctx['complete_frags_downloaded_bytes'] == 0:
+            write_flv_header(dest_stream)
+            if not live:
+                write_metadata_tag(dest_stream, metadata)
 
         base_url_parsed = compat_urllib_parse_urlparse(base_url)
 
         self._start_frag_download(ctx)
 
-        frags_filenames = []
+        frag_index = 0
         while fragments_list:
             seg_i, frag_i = fragments_list.pop(0)
+            frag_index += 1
+            if frag_index <= ctx['frag_index']:
+                continue
             name = 'Seg%d-Frag%d' % (seg_i, frag_i)
             query = []
             if base_url_parsed.query:
@@ -386,17 +387,10 @@ class F4mFD(FragmentFD):
             if info_dict.get('extra_param_to_segment_url'):
                 query.append(info_dict['extra_param_to_segment_url'])
             url_parsed = base_url_parsed._replace(path=base_url_parsed.path + name, query='&'.join(query))
-            frag_filename = '%s-%s' % (ctx['tmpfilename'], name)
             try:
-                success = ctx['dl'].download(frag_filename, {
-                    'url': url_parsed.geturl(),
-                    'http_headers': info_dict.get('http_headers'),
-                })
+                success, down_data = self._download_fragment(ctx, url_parsed.geturl(), info_dict)
                 if not success:
                     return False
-                (down, frag_sanitized) = sanitize_open(frag_filename, 'rb')
-                down_data = down.read()
-                down.close()
                 reader = FlvReader(down_data)
                 while True:
                     try:
@@ -411,12 +405,8 @@ class F4mFD(FragmentFD):
                             break
                         raise
                     if box_type == b'mdat':
-                        dest_stream.write(box_data)
+                        self._append_fragment(ctx, box_data)
                         break
-                if live:
-                    os.remove(encodeFilename(frag_sanitized))
-                else:
-                    frags_filenames.append(frag_sanitized)
             except (compat_urllib_error.HTTPError, ) as err:
                 if live and (err.code == 404 or err.code == 410):
                     # We didn't keep up with the live window. Continue
@@ -436,7 +426,4 @@ class F4mFD(FragmentFD):
 
         self._finish_frag_download(ctx)
 
-        for frag_file in frags_filenames:
-            os.remove(encodeFilename(frag_file))
-
         return True
diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 56f975266..44a3c1040 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -2,6 +2,7 @@ from __future__ import division, unicode_literals
 
 import os
 import time
+import io
 
 from .common import FileDownloader
 from .http import HttpFD
@@ -10,6 +11,7 @@ from ..utils import (
     encodeFilename,
     sanitize_open,
     sanitized_Request,
+    compat_str,
 )
 
 
@@ -30,13 +32,13 @@ class FragmentFD(FileDownloader):
                         Skip unavailable fragments (DASH and hlsnative only)
     """
 
-    def report_retry_fragment(self, err, fragment_name, count, retries):
+    def report_retry_fragment(self, err, frag_index, count, retries):
         self.to_screen(
-            '[download] Got server HTTP error: %s. Retrying fragment %s (attempt %d of %s)...'
-            % (error_to_compat_str(err), fragment_name, count, self.format_retries(retries)))
+            '[download] Got server HTTP error: %s. Retrying fragment %d (attempt %d of %s)...'
+            % (error_to_compat_str(err), frag_index, count, self.format_retries(retries)))
 
-    def report_skip_fragment(self, fragment_name):
-        self.to_screen('[download] Skipping fragment %s...' % fragment_name)
+    def report_skip_fragment(self, frag_index):
+        self.to_screen('[download] Skipping fragment %d...' % frag_index)
 
     def _prepare_url(self, info_dict, url):
         headers = info_dict.get('http_headers')
@@ -46,6 +48,25 @@ class FragmentFD(FileDownloader):
         self._prepare_frag_download(ctx)
         self._start_frag_download(ctx)
 
+    def _download_fragment(self, ctx, frag_url, info_dict, headers=None):
+        down = io.BytesIO()
+        success = ctx['dl'].download(down, {
+            'url': frag_url,
+            'http_headers': headers or info_dict.get('http_headers'),
+        })
+        if not success:
+            return False, None
+        frag_content = down.getvalue()
+        down.close()
+        return True, frag_content
+
+    def _append_fragment(self, ctx, frag_content):
+        ctx['dest_stream'].write(frag_content)
+        if not (ctx.get('live') or ctx['tmpfilename'] == '-'):
+            frag_index_stream, _ = sanitize_open(ctx['tmpfilename'] + '.fragindex', 'w')
+            frag_index_stream.write(compat_str(ctx['frag_index']))
+            frag_index_stream.close()
+
     def _prepare_frag_download(self, ctx):
         if 'live' not in ctx:
             ctx['live'] = False
@@ -66,11 +87,26 @@ class FragmentFD(FileDownloader):
             }
         )
         tmpfilename = self.temp_name(ctx['filename'])
-        dest_stream, tmpfilename = sanitize_open(tmpfilename, 'wb')
+        open_mode = 'wb'
+        resume_len = 0
+        frag_index = 0
+        # Establish possible resume length
+        if os.path.isfile(encodeFilename(tmpfilename)):
+            open_mode = 'ab'
+            resume_len = os.path.getsize(encodeFilename(tmpfilename))
+            if os.path.isfile(encodeFilename(tmpfilename + '.fragindex')):
+                frag_index_stream, _ = sanitize_open(tmpfilename + '.fragindex', 'r')
+                frag_index = int(frag_index_stream.read())
+                frag_index_stream.close()
+        dest_stream, tmpfilename = sanitize_open(tmpfilename, open_mode)
+
         ctx.update({
             'dl': dl,
             'dest_stream': dest_stream,
             'tmpfilename': tmpfilename,
+            'frag_index': frag_index,
+            # Total complete fragments downloaded so far in bytes
+            'complete_frags_downloaded_bytes': resume_len,
         })
 
     def _start_frag_download(self, ctx):
@@ -79,8 +115,8 @@ class FragmentFD(FileDownloader):
         # hook
         state = {
             'status': 'downloading',
-            'downloaded_bytes': 0,
-            'frag_index': 0,
+            'downloaded_bytes': ctx['complete_frags_downloaded_bytes'],
+            'frag_index': ctx['frag_index'],
             'frag_count': total_frags,
             'filename': ctx['filename'],
             'tmpfilename': ctx['tmpfilename'],
@@ -89,8 +125,6 @@ class FragmentFD(FileDownloader):
         start = time.time()
         ctx.update({
             'started': start,
-            # Total complete fragments downloaded so far in bytes
-            'complete_frags_downloaded_bytes': 0,
             # Amount of fragment's bytes downloaded by the time of the previous
             # frag progress hook invocation
             'prev_frag_downloaded_bytes': 0,
@@ -111,6 +145,7 @@ class FragmentFD(FileDownloader):
 
             if s['status'] == 'finished':
                 state['frag_index'] += 1
+                ctx['frag_index'] = state['frag_index']
                 state['downloaded_bytes'] += frag_total_bytes - ctx['prev_frag_downloaded_bytes']
                 ctx['complete_frags_downloaded_bytes'] = state['downloaded_bytes']
                 ctx['prev_frag_downloaded_bytes'] = 0
@@ -132,6 +167,8 @@ class FragmentFD(FileDownloader):
 
     def _finish_frag_download(self, ctx):
         ctx['dest_stream'].close()
+        if os.path.isfile(encodeFilename(ctx['tmpfilename'] + '.fragindex')):
+            os.remove(encodeFilename(ctx['tmpfilename'] + '.fragindex'))
         elapsed = time.time() - ctx['started']
         self.try_rename(ctx['tmpfilename'], ctx['filename'])
         fsize = os.path.getsize(encodeFilename(ctx['filename']))
diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index d0a5f7ba4..9a87d7ca8 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import os.path
 import re
 import binascii
 try:
@@ -18,8 +17,6 @@ from ..compat import (
     compat_struct_pack,
 )
 from ..utils import (
-    encodeFilename,
-    sanitize_open,
     parse_m3u8_attributes,
     update_url_query,
 )
@@ -103,17 +100,18 @@ class HlsFD(FragmentFD):
         media_sequence = 0
         decrypt_info = {'METHOD': 'NONE'}
         byte_range = {}
-        frags_filenames = []
+        frag_index = 0
         for line in s.splitlines():
             line = line.strip()
             if line:
                 if not line.startswith('#'):
+                    frag_index += 1
+                    if frag_index <= ctx['frag_index']:
+                        continue
                     frag_url = (
                         line
                         if re.match(r'^https?://', line)
                         else compat_urlparse.urljoin(man_url, line))
-                    frag_name = 'Frag%d' % i
-                    frag_filename = '%s-%s' % (ctx['tmpfilename'], frag_name)
                     if extra_query:
                         frag_url = update_url_query(frag_url, extra_query)
                     count = 0
@@ -122,15 +120,10 @@ class HlsFD(FragmentFD):
                         headers['Range'] = 'bytes=%d-%d' % (byte_range['start'], byte_range['end'])
                     while count <= fragment_retries:
                         try:
-                            success = ctx['dl'].download(frag_filename, {
-                                'url': frag_url,
-                                'http_headers': headers,
-                            })
+                            success, frag_content = self._download_fragment(
+                                ctx, frag_url, info_dict, headers)
                             if not success:
                                 return False
-                            down, frag_sanitized = sanitize_open(frag_filename, 'rb')
-                            frag_content = down.read()
-                            down.close()
                             break
                         except compat_urllib_error.HTTPError as err:
                             # Unavailable (possibly temporary) fragments may be served.
@@ -139,28 +132,29 @@ class HlsFD(FragmentFD):
                             # https://github.com/rg3/youtube-dl/issues/10448).
                             count += 1
                             if count <= fragment_retries:
-                                self.report_retry_fragment(err, frag_name, count, fragment_retries)
+                                self.report_retry_fragment(err, frag_index, count, fragment_retries)
                     if count > fragment_retries:
                         if skip_unavailable_fragments:
                             i += 1
                             media_sequence += 1
-                            self.report_skip_fragment(frag_name)
+                            self.report_skip_fragment(frag_index)
                             continue
                         self.report_error(
                             'giving up after %s fragment retries' % fragment_retries)
                         return False
                     if decrypt_info['METHOD'] == 'AES-128':
                         iv = decrypt_info.get('IV') or compat_struct_pack('>8xq', media_sequence)
+                        decrypt_info['KEY'] = decrypt_info.get('KEY') or self.ydl.urlopen(decrypt_info['URI']).read()
                         frag_content = AES.new(
                             decrypt_info['KEY'], AES.MODE_CBC, iv).decrypt(frag_content)
-                    ctx['dest_stream'].write(frag_content)
-                    frags_filenames.append(frag_sanitized)
+                    self._append_fragment(ctx, frag_content)
                     # We only download the first fragment during the test
                     if test:
                         break
                     i += 1
                     media_sequence += 1
                 elif line.startswith('#EXT-X-KEY'):
+                    decrypt_url = decrypt_info.get('URI')
                     decrypt_info = parse_m3u8_attributes(line[11:])
                     if decrypt_info['METHOD'] == 'AES-128':
                         if 'IV' in decrypt_info:
@@ -170,7 +164,8 @@ class HlsFD(FragmentFD):
                                 man_url, decrypt_info['URI'])
                         if extra_query:
                             decrypt_info['URI'] = update_url_query(decrypt_info['URI'], extra_query)
-                        decrypt_info['KEY'] = self.ydl.urlopen(decrypt_info['URI']).read()
+                        if decrypt_url != decrypt_info['URI']:
+                            decrypt_info['KEY'] = None
                 elif line.startswith('#EXT-X-MEDIA-SEQUENCE'):
                     media_sequence = int(line[22:])
                 elif line.startswith('#EXT-X-BYTERANGE'):
@@ -183,7 +178,4 @@ class HlsFD(FragmentFD):
 
         self._finish_frag_download(ctx)
 
-        for frag_file in frags_filenames:
-            os.remove(encodeFilename(frag_file))
-
         return True
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index af405b950..2896a17af 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -20,10 +20,14 @@ from ..utils import (
 
 
 class HttpFD(FileDownloader):
-    def real_download(self, filename, info_dict):
+    def real_download(self, filename_or_stream, info_dict):
         url = info_dict['url']
-        tmpfilename = self.temp_name(filename)
+        filename = filename_or_stream
         stream = None
+        if hasattr(filename_or_stream, 'write'):
+            stream = filename_or_stream
+            filename = '-'
+        tmpfilename = self.temp_name(filename)
 
         # Do not include the Accept-Encoding header
         headers = {'Youtubedl-no-compression': 'True'}
diff --git a/youtube_dl/downloader/ism.py b/youtube_dl/downloader/ism.py
index 63a636cb7..9f0fc36b3 100644
--- a/youtube_dl/downloader/ism.py
+++ b/youtube_dl/downloader/ism.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import os
 import time
 import struct
 import binascii
@@ -8,10 +7,6 @@ import io
 
 from .fragment import FragmentFD
 from ..compat import compat_urllib_error
-from ..utils import (
-    sanitize_open,
-    encodeFilename,
-)
 
 
 u8 = struct.Struct(b'>B')
@@ -225,16 +220,15 @@ class IsmFD(FragmentFD):
 
         self._prepare_and_start_frag_download(ctx)
 
-        segments_filenames = []
-
         fragment_retries = self.params.get('fragment_retries', 0)
         skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
 
         track_written = False
+        frag_index = 0
         for i, segment in enumerate(segments):
-            segment_url = segment['url']
-            segment_name = 'Frag%d' % i
-            target_filename = '%s-%s' % (ctx['tmpfilename'], segment_name)
+            frag_index += 1
+            if frag_index <= ctx['frag_index']:
+                continue
             count = 0
             while count <= fragment_retries:
                 try:
@@ -242,33 +236,27 @@ class IsmFD(FragmentFD):
                         'url': segment_url,
                         'http_headers': info_dict.get('http_headers'),
                     })
+                    success, frag_content = self._download_fragment(ctx, segment['url'], info_dict)
                     if not success:
                         return False
-                    down, target_sanitized = sanitize_open(target_filename, 'rb')
-                    down_data = down.read()
                     if not track_written:
-                        tfhd_data = extract_box_data(down_data, [b'moof', b'traf', b'tfhd'])
+                        tfhd_data = extract_box_data(frag_content, [b'moof', b'traf', b'tfhd'])
                         info_dict['_download_params']['track_id'] = u32.unpack(tfhd_data[4:8])[0]
                         write_piff_header(ctx['dest_stream'], info_dict['_download_params'])
                         track_written = True
-                    ctx['dest_stream'].write(down_data)
-                    down.close()
-                    segments_filenames.append(target_sanitized)
+                    self._append_fragment(ctx, frag_content)
                     break
                 except compat_urllib_error.HTTPError as err:
                     count += 1
                     if count <= fragment_retries:
-                        self.report_retry_fragment(err, segment_name, count, fragment_retries)
+                        self.report_retry_fragment(err, frag_index, count, fragment_retries)
             if count > fragment_retries:
                 if skip_unavailable_fragments:
-                    self.report_skip_fragment(segment_name)
+                    self.report_skip_fragment(frag_index)
                     continue
                 self.report_error('giving up after %s fragment retries' % fragment_retries)
                 return False
 
         self._finish_frag_download(ctx)
 
-        for segment_file in segments_filenames:
-            os.remove(encodeFilename(segment_file))
-
         return True
-- 
cgit 1.4.1


From ea0c2f219c219de8f59f1ae82e106cec5911c56c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 19 Apr 2017 18:34:25 +0100
Subject: [downloader/fragment] use a general file to store fragment download
 context

---
 youtube_dl/downloader/common.py   |  3 +++
 youtube_dl/downloader/fragment.py | 19 ++++++++++++-------
 2 files changed, 15 insertions(+), 7 deletions(-)

(limited to 'youtube_dl/downloader/fragment.py')

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index fdb77b620..5d6621147 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -187,6 +187,9 @@ class FileDownloader(object):
             return filename[:-len('.part')]
         return filename
 
+    def ytdl_filename(self, filename):
+        return filename + '.ytdl'
+
     def try_rename(self, old_filename, new_filename):
         try:
             if old_filename == new_filename:
diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 44a3c1040..80bb14d61 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -3,6 +3,7 @@ from __future__ import division, unicode_literals
 import os
 import time
 import io
+import json
 
 from .common import FileDownloader
 from .http import HttpFD
@@ -63,8 +64,10 @@ class FragmentFD(FileDownloader):
     def _append_fragment(self, ctx, frag_content):
         ctx['dest_stream'].write(frag_content)
         if not (ctx.get('live') or ctx['tmpfilename'] == '-'):
-            frag_index_stream, _ = sanitize_open(ctx['tmpfilename'] + '.fragindex', 'w')
-            frag_index_stream.write(compat_str(ctx['frag_index']))
+            frag_index_stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'w')
+            frag_index_stream.write(json.dumps({
+                'frag_index': ctx['frag_index']
+            }))
             frag_index_stream.close()
 
     def _prepare_frag_download(self, ctx):
@@ -94,9 +97,10 @@ class FragmentFD(FileDownloader):
         if os.path.isfile(encodeFilename(tmpfilename)):
             open_mode = 'ab'
             resume_len = os.path.getsize(encodeFilename(tmpfilename))
-            if os.path.isfile(encodeFilename(tmpfilename + '.fragindex')):
-                frag_index_stream, _ = sanitize_open(tmpfilename + '.fragindex', 'r')
-                frag_index = int(frag_index_stream.read())
+            ytdl_filename = encodeFilename(self.ytdl_filename(ctx['filename']))
+            if os.path.isfile(ytdl_filename):
+                frag_index_stream, _ = sanitize_open(ytdl_filename, 'r')
+                frag_index = json.loads(frag_index_stream.read())['frag_index']
                 frag_index_stream.close()
         dest_stream, tmpfilename = sanitize_open(tmpfilename, open_mode)
 
@@ -167,8 +171,9 @@ class FragmentFD(FileDownloader):
 
     def _finish_frag_download(self, ctx):
         ctx['dest_stream'].close()
-        if os.path.isfile(encodeFilename(ctx['tmpfilename'] + '.fragindex')):
-            os.remove(encodeFilename(ctx['tmpfilename'] + '.fragindex'))
+        ytdl_filename = encodeFilename(self.ytdl_filename(ctx['filename']))
+        if os.path.isfile(ytdl_filename):
+            os.remove(ytdl_filename)
         elapsed = time.time() - ctx['started']
         self.try_rename(ctx['tmpfilename'], ctx['filename'])
         fsize = os.path.getsize(encodeFilename(ctx['filename']))
-- 
cgit 1.4.1


From 3e0304fe6e3a194cfb04f21aa261effb0850da40 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 22 Apr 2017 16:42:24 +0100
Subject: [downloader/fragment] use the documented names for fragment
 progress_hooks fields

---
 youtube_dl/downloader/dash.py     |  2 +-
 youtube_dl/downloader/f4m.py      |  2 +-
 youtube_dl/downloader/fragment.py | 18 ++++++++++--------
 youtube_dl/downloader/hls.py      |  2 +-
 youtube_dl/downloader/ism.py      |  2 +-
 5 files changed, 14 insertions(+), 12 deletions(-)

(limited to 'youtube_dl/downloader/fragment.py')

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index 94a13a543..7491fdad8 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -28,7 +28,7 @@ class DashSegmentsFD(FragmentFD):
         frag_index = 0
         for i, segment in enumerate(segments):
             frag_index += 1
-            if frag_index <= ctx['frag_index']:
+            if frag_index <= ctx['fragment_index']:
                 continue
             # In DASH, the first segment contains necessary headers to
             # generate a valid MP4 file, so always abort for the first segment
diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index e456ed58f..c8fde9a89 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -376,7 +376,7 @@ class F4mFD(FragmentFD):
         while fragments_list:
             seg_i, frag_i = fragments_list.pop(0)
             frag_index += 1
-            if frag_index <= ctx['frag_index']:
+            if frag_index <= ctx['fragment_index']:
                 continue
             name = 'Seg%d-Frag%d' % (seg_i, frag_i)
             query = []
diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 80bb14d61..6c02cfc98 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -66,7 +66,9 @@ class FragmentFD(FileDownloader):
         if not (ctx.get('live') or ctx['tmpfilename'] == '-'):
             frag_index_stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'w')
             frag_index_stream.write(json.dumps({
-                'frag_index': ctx['frag_index']
+                'download': {
+                    'last_fragment_index': ctx['fragment_index']
+                },
             }))
             frag_index_stream.close()
 
@@ -100,7 +102,7 @@ class FragmentFD(FileDownloader):
             ytdl_filename = encodeFilename(self.ytdl_filename(ctx['filename']))
             if os.path.isfile(ytdl_filename):
                 frag_index_stream, _ = sanitize_open(ytdl_filename, 'r')
-                frag_index = json.loads(frag_index_stream.read())['frag_index']
+                frag_index = json.loads(frag_index_stream.read())['download']['last_fragment_index']
                 frag_index_stream.close()
         dest_stream, tmpfilename = sanitize_open(tmpfilename, open_mode)
 
@@ -108,7 +110,7 @@ class FragmentFD(FileDownloader):
             'dl': dl,
             'dest_stream': dest_stream,
             'tmpfilename': tmpfilename,
-            'frag_index': frag_index,
+            'fragment_index': frag_index,
             # Total complete fragments downloaded so far in bytes
             'complete_frags_downloaded_bytes': resume_len,
         })
@@ -120,8 +122,8 @@ class FragmentFD(FileDownloader):
         state = {
             'status': 'downloading',
             'downloaded_bytes': ctx['complete_frags_downloaded_bytes'],
-            'frag_index': ctx['frag_index'],
-            'frag_count': total_frags,
+            'fragment_index': ctx['fragment_index'],
+            'fragment_count': total_frags,
             'filename': ctx['filename'],
             'tmpfilename': ctx['tmpfilename'],
         }
@@ -144,12 +146,12 @@ class FragmentFD(FileDownloader):
             if not ctx['live']:
                 estimated_size = (
                     (ctx['complete_frags_downloaded_bytes'] + frag_total_bytes) /
-                    (state['frag_index'] + 1) * total_frags)
+                    (state['fragment_index'] + 1) * total_frags)
                 state['total_bytes_estimate'] = estimated_size
 
             if s['status'] == 'finished':
-                state['frag_index'] += 1
-                ctx['frag_index'] = state['frag_index']
+                state['fragment_index'] += 1
+                ctx['fragment_index'] = state['fragment_index']
                 state['downloaded_bytes'] += frag_total_bytes - ctx['prev_frag_downloaded_bytes']
                 ctx['complete_frags_downloaded_bytes'] = state['downloaded_bytes']
                 ctx['prev_frag_downloaded_bytes'] = 0
diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 9a87d7ca8..0e29c8a2a 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -106,7 +106,7 @@ class HlsFD(FragmentFD):
             if line:
                 if not line.startswith('#'):
                     frag_index += 1
-                    if frag_index <= ctx['frag_index']:
+                    if frag_index <= ctx['fragment_index']:
                         continue
                     frag_url = (
                         line
diff --git a/youtube_dl/downloader/ism.py b/youtube_dl/downloader/ism.py
index 9f0fc36b3..338820e71 100644
--- a/youtube_dl/downloader/ism.py
+++ b/youtube_dl/downloader/ism.py
@@ -227,7 +227,7 @@ class IsmFD(FragmentFD):
         frag_index = 0
         for i, segment in enumerate(segments):
             frag_index += 1
-            if frag_index <= ctx['frag_index']:
+            if frag_index <= ctx['fragment_index']:
                 continue
             count = 0
             while count <= fragment_retries:
-- 
cgit 1.4.1


From 4abdba643cc2d610fbbfe7ece9ee97a3df98ad33 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 22 Apr 2017 18:19:47 +0100
Subject: [downloader/fragment] remove unused code

---
 youtube_dl/downloader/fragment.py | 1 -
 youtube_dl/downloader/ism.py      | 4 ----
 2 files changed, 5 deletions(-)

(limited to 'youtube_dl/downloader/fragment.py')

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 6c02cfc98..0a3b1ece0 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -12,7 +12,6 @@ from ..utils import (
     encodeFilename,
     sanitize_open,
     sanitized_Request,
-    compat_str,
 )
 
 
diff --git a/youtube_dl/downloader/ism.py b/youtube_dl/downloader/ism.py
index 338820e71..5f6f9faef 100644
--- a/youtube_dl/downloader/ism.py
+++ b/youtube_dl/downloader/ism.py
@@ -232,10 +232,6 @@ class IsmFD(FragmentFD):
             count = 0
             while count <= fragment_retries:
                 try:
-                    success = ctx['dl'].download(target_filename, {
-                        'url': segment_url,
-                        'http_headers': info_dict.get('http_headers'),
-                    })
                     success, frag_content = self._download_fragment(ctx, segment['url'], info_dict)
                     if not success:
                         return False
-- 
cgit 1.4.1


From d3f0687cf7b049b976420056e02c26b5d96adeed Mon Sep 17 00:00:00 2001
From: Sergey M․ <dstftw@gmail.com>
Date: Mon, 24 Apr 2017 02:54:17 +0700
Subject: [downloader/fragment] Use temp file for current fragment

---
 youtube_dl/downloader/fragment.py | 59 ++++++++++++++++++++++++++-------------
 youtube_dl/downloader/http.py     |  8 ++----
 2 files changed, 41 insertions(+), 26 deletions(-)

(limited to 'youtube_dl/downloader/fragment.py')

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 0a3b1ece0..62de0a75b 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -2,7 +2,6 @@ from __future__ import division, unicode_literals
 
 import os
 import time
-import io
 import json
 
 from .common import FileDownloader
@@ -48,28 +47,42 @@ class FragmentFD(FileDownloader):
         self._prepare_frag_download(ctx)
         self._start_frag_download(ctx)
 
+    def _read_ytdl_file(self, ctx):
+        stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'r')
+        ctx['fragment_index'] = json.loads(stream.read())['download']['current_fragment_index']
+        stream.close()
+
+    def _write_ytdl_file(self, ctx):
+        frag_index_stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'w')
+        frag_index_stream.write(json.dumps({
+            'download': {
+                'current_fragment_index': ctx['fragment_index']
+            },
+        }))
+        frag_index_stream.close()
+
     def _download_fragment(self, ctx, frag_url, info_dict, headers=None):
-        down = io.BytesIO()
-        success = ctx['dl'].download(down, {
+        fragment_filename = '%s-Frag%d' % (ctx['tmpfilename'], ctx['fragment_index'])
+        success = ctx['dl'].download(fragment_filename, {
             'url': frag_url,
             'http_headers': headers or info_dict.get('http_headers'),
         })
         if not success:
             return False, None
-        frag_content = down.getvalue()
+        down, frag_sanitized = sanitize_open(fragment_filename, 'rb')
+        ctx['fragment_filename_sanitized'] = frag_sanitized
+        frag_content = down.read()
         down.close()
         return True, frag_content
 
     def _append_fragment(self, ctx, frag_content):
-        ctx['dest_stream'].write(frag_content)
-        if not (ctx.get('live') or ctx['tmpfilename'] == '-'):
-            frag_index_stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'w')
-            frag_index_stream.write(json.dumps({
-                'download': {
-                    'last_fragment_index': ctx['fragment_index']
-                },
-            }))
-            frag_index_stream.close()
+        try:
+            ctx['dest_stream'].write(frag_content)
+        finally:
+            if not (ctx.get('live') or ctx['tmpfilename'] == '-'):
+                self._write_ytdl_file(ctx)
+            os.remove(ctx['fragment_filename_sanitized'])
+            del ctx['fragment_filename_sanitized']
 
     def _prepare_frag_download(self, ctx):
         if 'live' not in ctx:
@@ -93,23 +106,29 @@ class FragmentFD(FileDownloader):
         tmpfilename = self.temp_name(ctx['filename'])
         open_mode = 'wb'
         resume_len = 0
-        frag_index = 0
+
         # Establish possible resume length
         if os.path.isfile(encodeFilename(tmpfilename)):
             open_mode = 'ab'
             resume_len = os.path.getsize(encodeFilename(tmpfilename))
-            ytdl_filename = encodeFilename(self.ytdl_filename(ctx['filename']))
-            if os.path.isfile(ytdl_filename):
-                frag_index_stream, _ = sanitize_open(ytdl_filename, 'r')
-                frag_index = json.loads(frag_index_stream.read())['download']['last_fragment_index']
-                frag_index_stream.close()
+
+        ctx['fragment_index'] = 0
+        if os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename']))):
+            self._read_ytdl_file(ctx)
+        else:
+            self._write_ytdl_file(ctx)
+
+        if ctx['fragment_index'] > 0:
+            assert resume_len > 0
+        else:
+            assert resume_len == 0
+
         dest_stream, tmpfilename = sanitize_open(tmpfilename, open_mode)
 
         ctx.update({
             'dl': dl,
             'dest_stream': dest_stream,
             'tmpfilename': tmpfilename,
-            'fragment_index': frag_index,
             # Total complete fragments downloaded so far in bytes
             'complete_frags_downloaded_bytes': resume_len,
         })
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 2896a17af..af405b950 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -20,14 +20,10 @@ from ..utils import (
 
 
 class HttpFD(FileDownloader):
-    def real_download(self, filename_or_stream, info_dict):
+    def real_download(self, filename, info_dict):
         url = info_dict['url']
-        filename = filename_or_stream
-        stream = None
-        if hasattr(filename_or_stream, 'write'):
-            stream = filename_or_stream
-            filename = '-'
         tmpfilename = self.temp_name(filename)
+        stream = None
 
         # Do not include the Accept-Encoding header
         headers = {'Youtubedl-no-compression': 'True'}
-- 
cgit 1.4.1


From 0eee52f34bfa55ba9d4ebfc4b4ba508c989f05b0 Mon Sep 17 00:00:00 2001
From: Sergey M․ <dstftw@gmail.com>
Date: Mon, 24 Apr 2017 03:09:08 +0700
Subject: Introduce --keep-fragments

---
 youtube_dl/__init__.py            | 1 +
 youtube_dl/downloader/fragment.py | 5 ++++-
 youtube_dl/options.py             | 4 ++++
 3 files changed, 9 insertions(+), 1 deletion(-)

(limited to 'youtube_dl/downloader/fragment.py')

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index f15606568..c4589411e 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -343,6 +343,7 @@ def _real_main(argv=None):
         'retries': opts.retries,
         'fragment_retries': opts.fragment_retries,
         'skip_unavailable_fragments': opts.skip_unavailable_fragments,
+        'keep_fragments': opts.keep_fragments,
         'buffersize': opts.buffersize,
         'noresizebuffer': opts.noresizebuffer,
         'continuedl': opts.continue_dl,
diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 62de0a75b..bcff94cbe 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -29,6 +29,8 @@ class FragmentFD(FileDownloader):
                         and hlsnative only)
     skip_unavailable_fragments:
                         Skip unavailable fragments (DASH and hlsnative only)
+    keep_fragments:     Keep downloaded fragments on disk after downloading is
+                        finished
     """
 
     def report_retry_fragment(self, err, frag_index, count, retries):
@@ -81,7 +83,8 @@ class FragmentFD(FileDownloader):
         finally:
             if not (ctx.get('live') or ctx['tmpfilename'] == '-'):
                 self._write_ytdl_file(ctx)
-            os.remove(ctx['fragment_filename_sanitized'])
+            if not self.params.get('keep_fragments', False):
+                os.remove(ctx['fragment_filename_sanitized'])
             del ctx['fragment_filename_sanitized']
 
     def _prepare_frag_download(self, ctx):
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 2d2f5e47b..52309fb84 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -468,6 +468,10 @@ def parseOpts(overrideArguments=None):
         '--abort-on-unavailable-fragment',
         action='store_false', dest='skip_unavailable_fragments',
         help='Abort downloading when some fragment is not available')
+    downloader.add_option(
+        '--keep-fragments',
+        action='store_true', dest='keep_fragments', default=False,
+        help='Keep downloaded fragments on disk after downloading is finished; fragments are erased by default')
     downloader.add_option(
         '--buffer-size',
         dest='buffersize', metavar='SIZE', default='1024',
-- 
cgit 1.4.1


From adb4b03cd583857d00fa03c050d1472bf44a0d15 Mon Sep 17 00:00:00 2001
From: Sergey M․ <dstftw@gmail.com>
Date: Mon, 24 Apr 2017 23:05:56 +0700
Subject: [downloader/fragment] Don't process ytdl file when it's not needed
 yet

---
 youtube_dl/downloader/fragment.py | 36 +++++++++++++++++++++++-------------
 1 file changed, 23 insertions(+), 13 deletions(-)

(limited to 'youtube_dl/downloader/fragment.py')

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index bcff94cbe..25c8f18ec 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -49,6 +49,10 @@ class FragmentFD(FileDownloader):
         self._prepare_frag_download(ctx)
         self._start_frag_download(ctx)
 
+    @staticmethod
+    def __do_ytdl_file(ctx):
+        return not ctx['live'] and not ctx['tmpfilename'] == '-'
+
     def _read_ytdl_file(self, ctx):
         stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'r')
         ctx['fragment_index'] = json.loads(stream.read())['download']['current_fragment_index']
@@ -81,7 +85,7 @@ class FragmentFD(FileDownloader):
         try:
             ctx['dest_stream'].write(frag_content)
         finally:
-            if not (ctx.get('live') or ctx['tmpfilename'] == '-'):
+            if self.__do_ytdl_file(ctx):
                 self._write_ytdl_file(ctx)
             if not self.params.get('keep_fragments', False):
                 os.remove(ctx['fragment_filename_sanitized'])
@@ -115,16 +119,21 @@ class FragmentFD(FileDownloader):
             open_mode = 'ab'
             resume_len = os.path.getsize(encodeFilename(tmpfilename))
 
-        ctx['fragment_index'] = 0
-        if os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename']))):
-            self._read_ytdl_file(ctx)
-        else:
-            self._write_ytdl_file(ctx)
+        # Should be initialized before ytdl file check
+        ctx.update({
+            'tmpfilename': tmpfilename,
+            'fragment_index': 0,
+        })
 
-        if ctx['fragment_index'] > 0:
-            assert resume_len > 0
-        else:
-            assert resume_len == 0
+        if self.__do_ytdl_file(ctx):
+            if os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename']))):
+                self._read_ytdl_file(ctx)
+            else:
+                self._write_ytdl_file(ctx)
+            if ctx['fragment_index'] > 0:
+                assert resume_len > 0
+            else:
+                assert resume_len == 0
 
         dest_stream, tmpfilename = sanitize_open(tmpfilename, open_mode)
 
@@ -194,9 +203,10 @@ class FragmentFD(FileDownloader):
 
     def _finish_frag_download(self, ctx):
         ctx['dest_stream'].close()
-        ytdl_filename = encodeFilename(self.ytdl_filename(ctx['filename']))
-        if os.path.isfile(ytdl_filename):
-            os.remove(ytdl_filename)
+        if self.__do_ytdl_file(ctx):
+            ytdl_filename = encodeFilename(self.ytdl_filename(ctx['filename']))
+            if os.path.isfile(ytdl_filename):
+                os.remove(ytdl_filename)
         elapsed = time.time() - ctx['started']
         self.try_rename(ctx['tmpfilename'], ctx['filename'])
         fsize = os.path.getsize(encodeFilename(ctx['filename']))
-- 
cgit 1.4.1


From 290f64dbaa3a8c187d47b72b3c808bd52d15a6e8 Mon Sep 17 00:00:00 2001
From: Sergey M․ <dstftw@gmail.com>
Date: Mon, 24 Apr 2017 23:50:20 +0700
Subject: [downloader/fragment] Improve .ytdl format and start documenting

---
 youtube_dl/downloader/fragment.py | 31 ++++++++++++++++++++++++++-----
 1 file changed, 26 insertions(+), 5 deletions(-)

(limited to 'youtube_dl/downloader/fragment.py')

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 25c8f18ec..50c8254a4 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -31,6 +31,24 @@ class FragmentFD(FileDownloader):
                         Skip unavailable fragments (DASH and hlsnative only)
     keep_fragments:     Keep downloaded fragments on disk after downloading is
                         finished
+
+    For each incomplete fragment download youtube-dl keeps on disk a special
+    bookkeeping file with download state and metadata (in future such files will
+    be used for any incomplete download handled by youtube-dl). This file is
+    used to properly handle resuming, check download file consistency and detect
+    potential errors. The file has a .ytdl extension and represents a standard
+    JSON file of the following format:
+
+    extractor:
+        Dictionary of extractor related data. TBD.
+
+    downloader:
+        Dictionary of downloader related data. May contain following data:
+            current_fragment:
+                Dictionary with current (being downloaded) fragment data:
+                index:  Index of current fragment among all fragments
+            fragment_count:
+                Total count of fragments
     """
 
     def report_retry_fragment(self, err, frag_index, count, retries):
@@ -55,16 +73,19 @@ class FragmentFD(FileDownloader):
 
     def _read_ytdl_file(self, ctx):
         stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'r')
-        ctx['fragment_index'] = json.loads(stream.read())['download']['current_fragment_index']
+        ctx['fragment_index'] = json.loads(stream.read())['downloader']['current_fragment']['index']
         stream.close()
 
     def _write_ytdl_file(self, ctx):
         frag_index_stream, _ = sanitize_open(self.ytdl_filename(ctx['filename']), 'w')
-        frag_index_stream.write(json.dumps({
-            'download': {
-                'current_fragment_index': ctx['fragment_index']
+        downloader = {
+            'current_fragment': {
+                'index': ctx['fragment_index'],
             },
-        }))
+        }
+        if ctx.get('fragment_count') is not None:
+            downloader['fragment_count'] = ctx['fragment_count']
+        frag_index_stream.write(json.dumps({'downloader': downloader}))
         frag_index_stream.close()
 
     def _download_fragment(self, ctx, frag_url, info_dict, headers=None):
-- 
cgit 1.4.1


From 85f6de25e46562f57be4447e9109be964e6515ea Mon Sep 17 00:00:00 2001
From: Sergey M․ <dstftw@gmail.com>
Date: Tue, 25 Apr 2017 23:33:35 +0700
Subject: [downloader/fragment] Clarify current_fragment's index and mark as
 experimental

---
 youtube_dl/downloader/fragment.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'youtube_dl/downloader/fragment.py')

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 50c8254a4..bb9e82578 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -46,9 +46,11 @@ class FragmentFD(FileDownloader):
         Dictionary of downloader related data. May contain following data:
             current_fragment:
                 Dictionary with current (being downloaded) fragment data:
-                index:  Index of current fragment among all fragments
+                index:  0-based index of current fragment among all fragments
             fragment_count:
                 Total count of fragments
+                
+    This feature is experimental and file format may change in future.
     """
 
     def report_retry_fragment(self, err, frag_index, count, retries):
-- 
cgit 1.4.1


From c0fa4245cea1dc493e5a72e36644114a2743039c Mon Sep 17 00:00:00 2001
From: Sergey M․ <dstftw@gmail.com>
Date: Fri, 28 Apr 2017 03:24:59 +0700
Subject: [downloader/fragment] Remove assert for resume_len when no fragments
 downloaded This may be incorrect due some header (e.g. flv header in f4m
 downloader)

---
 youtube_dl/downloader/fragment.py | 2 --
 1 file changed, 2 deletions(-)

(limited to 'youtube_dl/downloader/fragment.py')

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index bb9e82578..d529ae09a 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -155,8 +155,6 @@ class FragmentFD(FileDownloader):
                 self._write_ytdl_file(ctx)
             if ctx['fragment_index'] > 0:
                 assert resume_len > 0
-            else:
-                assert resume_len == 0
 
         dest_stream, tmpfilename = sanitize_open(tmpfilename, open_mode)
 
-- 
cgit 1.4.1


From 50534b7158e0e1c842d05eae39fd6687a04a76e3 Mon Sep 17 00:00:00 2001
From: Sergey M․ <dstftw@gmail.com>
Date: Sun, 30 Apr 2017 22:04:01 +0700
Subject: [downloader/fragment] PEP 8

---
 youtube_dl/downloader/fragment.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'youtube_dl/downloader/fragment.py')

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index d529ae09a..bccc8ecc1 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -49,7 +49,7 @@ class FragmentFD(FileDownloader):
                 index:  0-based index of current fragment among all fragments
             fragment_count:
                 Total count of fragments
-                
+
     This feature is experimental and file format may change in future.
     """
 
-- 
cgit 1.4.1


From e7c3e33456e155106ad08347d8ab9a2ecd0c8a11 Mon Sep 17 00:00:00 2001
From: Sergey M․ <dstftw@gmail.com>
Date: Thu, 14 Sep 2017 23:19:53 +0700
Subject: [downloader/fragment] Restart inconsistent incomplete fragment
 downloads (#13731)

---
 youtube_dl/downloader/fragment.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

(limited to 'youtube_dl/downloader/fragment.py')

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index bccc8ecc1..6f6fb4a77 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -151,10 +151,15 @@ class FragmentFD(FileDownloader):
         if self.__do_ytdl_file(ctx):
             if os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename']))):
                 self._read_ytdl_file(ctx)
+                if ctx['fragment_index'] > 0 and resume_len == 0:
+                    self.report_error(
+                        'Inconsistent state of incomplete fragment download. '
+                        'Restarting from the beginning...')
+                    ctx['fragment_index'] = resume_len = 0
+                    self._write_ytdl_file(ctx)
             else:
                 self._write_ytdl_file(ctx)
-            if ctx['fragment_index'] > 0:
-                assert resume_len > 0
+                assert ctx['fragment_index'] == 0
 
         dest_stream, tmpfilename = sanitize_open(tmpfilename, open_mode)
 
-- 
cgit 1.4.1


From 5efaf43c93247446c8616454bb3f59b7ae13fcb8 Mon Sep 17 00:00:00 2001
From: Sergey M․ <dstftw@gmail.com>
Date: Sun, 15 Oct 2017 06:13:07 +0700
Subject: [downloader/fragment] Output ad fragment count

---
 youtube_dl/downloader/fragment.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

(limited to 'youtube_dl/downloader/fragment.py')

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 6f6fb4a77..7e891b92a 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -117,9 +117,15 @@ class FragmentFD(FileDownloader):
     def _prepare_frag_download(self, ctx):
         if 'live' not in ctx:
             ctx['live'] = False
+        if not ctx['live']:
+            total_frags_str = '%d' % ctx['total_frags']
+            ad_frags = ctx.get('ad_frags', 0)
+            if ad_frags:
+                total_frags_str += ' (not including %d ad)' % ad_frags
+        else:
+            total_frags_str = 'unknown (live)'
         self.to_screen(
-            '[%s] Total fragments: %s'
-            % (self.FD_NAME, ctx['total_frags'] if not ctx['live'] else 'unknown (live)'))
+            '[%s] Total fragments: %s' % (self.FD_NAME, total_frags_str))
         self.report_destination(ctx['filename'])
         dl = HttpQuietDownloader(
             self.ydl,
-- 
cgit 1.4.1


From 6f3b4a98c96c54eb0e3933b3cd6b4e6848582a02 Mon Sep 17 00:00:00 2001
From: Sergey M․ <dstftw@gmail.com>
Date: Tue, 17 Oct 2017 22:53:34 +0700
Subject: [downloader/fragment] Report warning instead of error on inconsistent
 download state

---
 youtube_dl/downloader/fragment.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'youtube_dl/downloader/fragment.py')

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 7e891b92a..93002e45a 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -158,7 +158,7 @@ class FragmentFD(FileDownloader):
             if os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename']))):
                 self._read_ytdl_file(ctx)
                 if ctx['fragment_index'] > 0 and resume_len == 0:
-                    self.report_error(
+                    self.report_warning(
                         'Inconsistent state of incomplete fragment download. '
                         'Restarting from the beginning...')
                     ctx['fragment_index'] = resume_len = 0
-- 
cgit 1.4.1


From 593f2f798922d54be8d3f20f4a2048493095016d Mon Sep 17 00:00:00 2001
From: Sergey M․ <dstftw@gmail.com>
Date: Sat, 2 Dec 2017 21:15:45 +0700
Subject: [downloader/fragment] Commit part file after each fragment In order
 to obtain correct resume_len on next iteration

---
 youtube_dl/downloader/fragment.py | 1 +
 1 file changed, 1 insertion(+)

(limited to 'youtube_dl/downloader/fragment.py')

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 93002e45a..7bb61a541 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -107,6 +107,7 @@ class FragmentFD(FileDownloader):
     def _append_fragment(self, ctx, frag_content):
         try:
             ctx['dest_stream'].write(frag_content)
+            ctx['dest_stream'].flush()
         finally:
             if self.__do_ytdl_file(ctx):
                 self._write_ytdl_file(ctx)
-- 
cgit 1.4.1


From 99081da90c9b8ce4ee7fe9452787507fed4251a0 Mon Sep 17 00:00:00 2001
From: Sergey M․ <dstftw@gmail.com>
Date: Mon, 18 Dec 2017 03:31:53 +0700
Subject: [downloader/fragment] Encode filename of fragment being removed
 (closes #15020)

---
 youtube_dl/downloader/fragment.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'youtube_dl/downloader/fragment.py')

diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 7bb61a541..ea5e3a4b5 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -112,7 +112,7 @@ class FragmentFD(FileDownloader):
             if self.__do_ytdl_file(ctx):
                 self._write_ytdl_file(ctx)
             if not self.params.get('keep_fragments', False):
-                os.remove(ctx['fragment_filename_sanitized'])
+                os.remove(encodeFilename(ctx['fragment_filename_sanitized']))
             del ctx['fragment_filename_sanitized']
 
     def _prepare_frag_download(self, ctx):
-- 
cgit 1.4.1