From b096e610986870d24b5eb67fa118936642f41238 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Sun, 12 Feb 2017 08:26:36 -0500
Subject: [PATCH 01/22] [Brightcove] support bostonglobe.com's use

BrightcoveNewIE.extract_urls(): Handle player data when it is all in
attributes of the <video> tag, and also the id is
data-brightcove-video-id not data-video-id.

Add test to generic extractor, note HEAD gives 404, no checksum computed(?).
---
 youtube_dl/extractor/brightcove.py | 17 ++++++++++++++++-
 youtube_dl/extractor/generic.py    | 22 ++++++++++++++++++++++
 2 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 5c6e99da1..f1eeda88a 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -508,7 +508,22 @@ class BrightcoveNewIE(InfoExtractor):
                 'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
                 % (account_id, player_id, embed, video_id))
 
-        return entries
+# 	<video data-brightcove-video-id="5320421710001" data-account="245991542" data-player="SJWAiyYWg" data-embed="default" class="video-js" controls itemscope itemtype="http://schema.org/VideoObject">
+        for video_id, account_id, player_id, embed in re.findall(
+                r'''(?sx)
+                    <video[^>]+
+                        data-brightcove-video-id=["\'](\d+|ref:[^"\']+)["\'].*?
+                        data-account=["\'](\d+)["\'].*?
+                        data-player=["\'](\w+)["\'].*?
+                        data-embed=["\'](\w+)["\'].*?
+                    </video>
+                ''', webpage):
+            entries.append(
+                'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
+                % (account_id, player_id, embed, video_id))
+
+
+	return entries
 
     def _real_extract(self, url):
         account_id, player_id, embed, video_id = re.match(self._VALID_URL, url).groups()
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 1c233f038..4a422eaff 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -447,6 +447,28 @@ class GenericIE(InfoExtractor):
                 },
             }],
         },
+        {
+            # Brightcove with metadata on one line
+            'url': 'http://www.bostonglobe.com/metro/2017/02/11/tree-finally-succumbs-disease-leaving-hole-neighborhood/h1b4lviqzMTIn9sVy8F3gP/story.html?',
+            'info_dict': {
+                'id': 'story',
+                'title': 'A tree finally succumbs to disease, leaving a hole in a neighborhood - The Boston Globe',
+            },
+            'playlist': [{
+                'info_dict': {
+                    'id': '5320421710001',
+                    'ext': 'mp4',
+                    'title': 'A tree finally succumbs to disease, leaving a hole in a neighborhood',
+                    'description': 'It arrived as a sapling when the Back Bay was in its infancy, a spindly American elm tamped down into a square of dirt cut into the brick sidewalk of 1880s Marlborough Street, no higher than the first bay window of the new brownstone behind it.',
+                    'timestamp': 1486877593,
+                    'upload_date': '20170212',
+                    'uploader_id': '245991542',
+                },
+            }],
+            # HEAD requests produce 404 :(
+            'expected_warnings': ['404'],
+
+        },
         # ooyala video
         {
             'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',

From e94df37ad6da3a4690170af869e69e8a95c60226 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Sun, 12 Feb 2017 09:17:17 -0500
Subject: [PATCH 02/22] [Brightcove] Use extract_attributes, also handle
 data-video-id

Per @yan12125, use extract_attributes() since order may differ.

While preferring data-brightcove-video-id, use data-video-id if present,
to support #12005.
---
 youtube_dl/extractor/brightcove.py | 29 ++++++++++++++++-------------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index f1eeda88a..7cef3657d 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -17,6 +17,7 @@ from ..compat import (
 from ..utils import (
     determine_ext,
     ExtractorError,
+    extract_attributes,
     find_xpath_attr,
     fix_xml_ampersands,
     float_or_none,
@@ -508,20 +509,22 @@ class BrightcoveNewIE(InfoExtractor):
                 'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
                 % (account_id, player_id, embed, video_id))
 
-# 	<video data-brightcove-video-id="5320421710001" data-account="245991542" data-player="SJWAiyYWg" data-embed="default" class="video-js" controls itemscope itemtype="http://schema.org/VideoObject">
-        for video_id, account_id, player_id, embed in re.findall(
-                r'''(?sx)
-                    <video[^>]+
-                        data-brightcove-video-id=["\'](\d+|ref:[^"\']+)["\'].*?
-                        data-account=["\'](\d+)["\'].*?
-                        data-player=["\'](\w+)["\'].*?
-                        data-embed=["\'](\w+)["\'].*?
-                    </video>
-                ''', webpage):
-            entries.append(
-                'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
-                % (account_id, player_id, embed, video_id))
+        # 	<video data-brightcove-video-id="5320421710001" data-account="245991542" data-player="SJWAiyYWg" data-embed="default" class="video-js" controls itemscope itemtype="http://schema.org/VideoObject">
+        for video in re.findall(r'(?i)(<video[^>]+>)', webpage):
+            attrs = extract_attributes(video)
+                
+            video_id    = attrs.get('data-brightcove-video-id')
+            if video_id is None:
+                video_id = attrs.get('data-video-id')
 
+            account_id  = attrs.get('data-account')
+            player_id   = attrs.get('data-player')
+            embed       = attrs.get('data-embed')
+
+            if video_id and account_id and player_id and embed:
+                entries.append(
+                    'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
+                    % (account_id, player_id, embed, video_id))
 
 	return entries
 

From 3de370796e378dd20ea3d8c4c706b39d9360a832 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Sun, 12 Feb 2017 13:03:05 -0500
Subject: [PATCH 03/22] [Brightcove] dict_get not if .get(x): .get(y)

---
 youtube_dl/extractor/brightcove.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 7cef3657d..cdc3d6024 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -16,6 +16,7 @@ from ..compat import (
 )
 from ..utils import (
     determine_ext,
+    dict_get,
     ExtractorError,
     extract_attributes,
     find_xpath_attr,
@@ -513,10 +514,7 @@ class BrightcoveNewIE(InfoExtractor):
         for video in re.findall(r'(?i)(<video[^>]+>)', webpage):
             attrs = extract_attributes(video)
                 
-            video_id    = attrs.get('data-brightcove-video-id')
-            if video_id is None:
-                video_id = attrs.get('data-video-id')
-
+            video_id    = dict_get(attrs, ['data-brightcove-video-id', 'data-video-id'])
             account_id  = attrs.get('data-account')
             player_id   = attrs.get('data-player')
             embed       = attrs.get('data-embed')

From 101118632754e6a3e5d7aa27c9f708c5f6769bb9 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Mon, 13 Feb 2017 09:06:02 -0500
Subject: [PATCH 04/22] [Brightcove] whitespace fixup (travis)

---
 youtube_dl/extractor/brightcove.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index cdc3d6024..5e482c7ba 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -524,7 +524,7 @@ class BrightcoveNewIE(InfoExtractor):
                     'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
                     % (account_id, player_id, embed, video_id))
 
-	return entries
+        return entries
 
     def _real_extract(self, url):
         account_id, player_id, embed, video_id = re.match(self._VALID_URL, url).groups()

From a5a2487d58f1a6a591a4d38fcd21b0fae7182681 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Sun, 12 Feb 2017 13:14:01 -0500
Subject: [PATCH 05/22] [Brightcove]/generic.py: Another test

---
 youtube_dl/extractor/generic.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 4a422eaff..19aa78565 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -467,7 +467,26 @@ class GenericIE(InfoExtractor):
             }],
             # HEAD requests produce 404 :(
             'expected_warnings': ['404'],
-
+        },
+        {
+            # Alternative brightcove <video> attributes
+            'url': 'http://www.programme-tv.net/videos/extraits/81095-guillaume-canet-evoque-les-rumeurs-d-infidelite-de-marion-cotillard-avec-brad-pitt-dans-vivement-dimanche/',
+            'info_dict': {
+                'id': '81095-guillaume-canet-evoque-les-rumeurs-d-infidelite-de-marion-cotillard-avec-brad-pitt-dans-vivement-dimanche',
+                'title': "Guillaume Canet évoque les rumeurs d'infidélité de Marion Cotillard avec Brad Pitt dans Vivement Dimanche, Extraits : toutes les vidéos avec Télé-Loisirs",
+            },
+            'playlist': [{
+                'info_dict': {
+                    'id': '5311302538001',
+                    'ext': 'mp4',
+                    'title': "Guillaume Canet évoque les rumeurs d'infidélité de Marion Cotillard avec Brad Pitt dans Vivement Dimanche",
+                    'description': "Guillaume Canet évoque les rumeurs d'infidélité de Marion Cotillard avec Brad Pitt dans Vivement Dimanche (France 2, 5 février 2017)",
+                    'timestamp': 1486321708,
+                    'upload_date': '20170205',
+                    'uploader_id': '800000640001',
+                },
+            'only_matching': True,
+            }],
         },
         # ooyala video
         {

From 20556c24973cc1dd1c5cfcb72d136a606f6d38c6 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Sat, 18 Feb 2017 21:03:12 -0500
Subject: [PATCH 06/22] cleanup remnants from cherry-pick

---
 youtube_dl/extractor/brightcove.py |  3 +--
 youtube_dl/extractor/generic.py    | 21 ---------------------
 2 files changed, 1 insertion(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 5e482c7ba..dea38a8d7 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -16,7 +16,6 @@ from ..compat import (
 )
 from ..utils import (
     determine_ext,
-    dict_get,
     ExtractorError,
     extract_attributes,
     find_xpath_attr,
@@ -514,7 +513,7 @@ class BrightcoveNewIE(InfoExtractor):
         for video in re.findall(r'(?i)(<video[^>]+>)', webpage):
             attrs = extract_attributes(video)
                 
-            video_id    = dict_get(attrs, ['data-brightcove-video-id', 'data-video-id'])
+            video_id    = attrs.get('data-video-id')
             account_id  = attrs.get('data-account')
             player_id   = attrs.get('data-player')
             embed       = attrs.get('data-embed')
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 19aa78565..4a571956f 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -447,27 +447,6 @@ class GenericIE(InfoExtractor):
                 },
             }],
         },
-        {
-            # Brightcove with metadata on one line
-            'url': 'http://www.bostonglobe.com/metro/2017/02/11/tree-finally-succumbs-disease-leaving-hole-neighborhood/h1b4lviqzMTIn9sVy8F3gP/story.html?',
-            'info_dict': {
-                'id': 'story',
-                'title': 'A tree finally succumbs to disease, leaving a hole in a neighborhood - The Boston Globe',
-            },
-            'playlist': [{
-                'info_dict': {
-                    'id': '5320421710001',
-                    'ext': 'mp4',
-                    'title': 'A tree finally succumbs to disease, leaving a hole in a neighborhood',
-                    'description': 'It arrived as a sapling when the Back Bay was in its infancy, a spindly American elm tamped down into a square of dirt cut into the brick sidewalk of 1880s Marlborough Street, no higher than the first bay window of the new brownstone behind it.',
-                    'timestamp': 1486877593,
-                    'upload_date': '20170212',
-                    'uploader_id': '245991542',
-                },
-            }],
-            # HEAD requests produce 404 :(
-            'expected_warnings': ['404'],
-        },
         {
             # Alternative brightcove <video> attributes
             'url': 'http://www.programme-tv.net/videos/extraits/81095-guillaume-canet-evoque-les-rumeurs-d-infidelite-de-marion-cotillard-avec-brad-pitt-dans-vivement-dimanche/',

From 4cb942263a68dc1f01677055cda9dd290fa512be Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Sat, 18 Feb 2017 21:03:51 -0500
Subject: [PATCH 07/22] whitespace fix

---
 youtube_dl/extractor/brightcove.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index dea38a8d7..a80a83f29 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -512,7 +512,7 @@ class BrightcoveNewIE(InfoExtractor):
         # 	<video data-brightcove-video-id="5320421710001" data-account="245991542" data-player="SJWAiyYWg" data-embed="default" class="video-js" controls itemscope itemtype="http://schema.org/VideoObject">
         for video in re.findall(r'(?i)(<video[^>]+>)', webpage):
             attrs = extract_attributes(video)
-                
+
             video_id    = attrs.get('data-video-id')
             account_id  = attrs.get('data-account')
             player_id   = attrs.get('data-player')

From 50c71279c3422534f5c2721d29d9c0d1e4013ed9 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Mon, 6 Mar 2017 22:41:44 -0500
Subject: [PATCH 08/22] [Generic] Test for existing [Brightcove:new]/[2] that
 has no test

---
 youtube_dl/extractor/generic.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 4a571956f..d0eb5d419 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -447,6 +447,20 @@ class GenericIE(InfoExtractor):
                 },
             }],
         },
+        {
+            # Brightcove:new type [2].
+            'url': 'http://www.delawaresportszone.com/video-st-thomas-more-earns-first-trip-to-basketball-semis',
+            'md5': '2b35148fcf48da41c9fb4591650784f3',
+            'info_dict': {
+                'id': '5348741021001',
+                'ext': 'mp4',
+                'upload_date': '20170306',
+                'uploader_id': '4191638492001',
+                'timestamp': 1488769918,
+                'title': 'VIDEO:  St. Thomas More earns first trip to basketball semis',
+
+            },
+        },
         {
             # Alternative brightcove <video> attributes
             'url': 'http://www.programme-tv.net/videos/extraits/81095-guillaume-canet-evoque-les-rumeurs-d-infidelite-de-marion-cotillard-avec-brad-pitt-dans-vivement-dimanche/',

From 976a38db62a26a6020f4f04b2191727045bb9751 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Mon, 6 Mar 2017 22:43:00 -0500
Subject: [PATCH 09/22] [generic] Treat both Brightcove's with similar
 verbosity

And distinguish then with
    (legacy)
and
    (new)
in the output.
---
 youtube_dl/extractor/generic.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index d0eb5d419..785f555fa 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1848,7 +1848,7 @@ class GenericIE(InfoExtractor):
         # Look for Brightcove Legacy Studio embeds
         bc_urls = BrightcoveLegacyIE._extract_brightcove_urls(webpage)
         if bc_urls:
-            self.to_screen('Brightcove video detected.')
+            self.to_screen('Brightcove (legacy) video detected.')
             entries = [{
                 '_type': 'url',
                 'url': smuggle_url(bc_url, {'Referer': url}),
@@ -1865,6 +1865,7 @@ class GenericIE(InfoExtractor):
         # Look for Brightcove New Studio embeds
         bc_urls = BrightcoveNewIE._extract_urls(webpage)
         if bc_urls:
+            self.to_screen('Brightcove (new) video detected.')
             return _playlist_from_matches(bc_urls, ie='BrightcoveNew')
 
         # Look for ThePlatform embeds

From 68ba9882a3c3bedb0e5a866d5b77e71d0ad2c885 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Mon, 6 Mar 2017 22:45:43 -0500
Subject: [PATCH 10/22] [Generic] missing hash for brightcove programme-tv test

---
 youtube_dl/extractor/generic.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 785f555fa..91964219f 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -469,6 +469,7 @@ class GenericIE(InfoExtractor):
                 'title': "Guillaume Canet évoque les rumeurs d'infidélité de Marion Cotillard avec Brad Pitt dans Vivement Dimanche, Extraits : toutes les vidéos avec Télé-Loisirs",
             },
             'playlist': [{
+                'md5': '732d22ba3d33f2f3fc253c39f8f36523',
                 'info_dict': {
                     'id': '5311302538001',
                     'ext': 'mp4',

From 49571c1c2fc0872d8e8cf341cdeacd57d8885236 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Tue, 7 Mar 2017 01:00:42 -0500
Subject: [PATCH 11/22] [brightcove:new] Combine <video> tag searches

Per @yan12125.

Instead of searching for <video> followed by <script>, now
search for <video> optionally followed by <script>. If we
find <script>, use it. Otherwise, extract_attributes() from the video
tag and use those.

Move stripping the 'ref': prefix of data-video-id="ref:12345" out of
regexp into code, since we now use extract_attributes().
---
 youtube_dl/extractor/brightcove.py | 49 ++++++++++++++++--------------
 1 file changed, 26 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index a80a83f29..2cb1bca8c 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -482,8 +482,12 @@ class BrightcoveNewIE(InfoExtractor):
         # Reference:
         # 1. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideoiniframe
         # 2. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideousingjavascript
-        # 3. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/embed-in-page.html
+        # 3. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/embed-in-page.html 
         # 4. https://support.brightcove.com/en/video-cloud/docs/dynamically-assigning-videos-player
+        # 5. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#tag
+
+        # [5] looks like:
+        # 	<video data-video-id="5320421710001" data-account="245991542" data-player="SJWAiyYWg" data-embed="default" class="video-js" controls itemscope itemtype="http://schema.org/VideoObject">
 
         entries = []
 
@@ -492,31 +496,30 @@ class BrightcoveNewIE(InfoExtractor):
                 r'<iframe[^>]+src=(["\'])((?:https?:)?//players\.brightcove\.net/\d+/[^/]+/index\.html.+?)\1', webpage):
             entries.append(url if url.startswith('http') else 'http:' + url)
 
-        # Look for embed_in_page embeds [2]
-        for video_id, account_id, player_id, embed in re.findall(
-                # According to examples from [3] it's unclear whether video id
-                # may be optional and what to do when it is
-                # According to [4] data-video-id may be prefixed with ref:
-                r'''(?sx)
-                    <video[^>]+
-                        data-video-id=["\'](\d+|ref:[^"\']+)["\'][^>]*>.*?
-                    </video>.*?
-                    <script[^>]+
-                        src=["\'](?:https?:)?//players\.brightcove\.net/
-                        (\d+)/([^/]+)_([^/]+)/index(?:\.min)?\.js
-                ''', webpage):
-            entries.append(
-                'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
-                % (account_id, player_id, embed, video_id))
-
-        # 	<video data-brightcove-video-id="5320421710001" data-account="245991542" data-player="SJWAiyYWg" data-embed="default" class="video-js" controls itemscope itemtype="http://schema.org/VideoObject">
-        for video in re.findall(r'(?i)(<video[^>]+>)', webpage):
+        # Look for embed_in_page embeds [2] and <video> tags [5]
+        for video, script_tag, account_id, player_id, embed in re.findall(
+        r'''(?isx)
+            (<video[^>]+>)
+            (?:.*?
+              (<script[^>]+
+                src=["\'](?:https?:)?//players\.brightcove\.net/
+                (\d+)/([^/]+)_([^/]+)/index(?:\.min)?\.js
+              )
+            )?
+        ''', webpage):
             attrs = extract_attributes(video)
 
+            # According to examples from [3] it's unclear whether video id
+            # may be optional and what to do when it is
             video_id    = attrs.get('data-video-id')
-            account_id  = attrs.get('data-account')
-            player_id   = attrs.get('data-player')
-            embed       = attrs.get('data-embed')
+            # See PR#12099/bostonglobe.py for 'data-brightcove-video-id' variant
+
+            if not account_id: account_id  = attrs.get('data-account')
+            if not player_id:  player_id   = attrs.get('data-player')
+            if not embed:      embed       = attrs.get('data-embed')
+
+            # According to [4] data-video-id may be prefixed with 'ref:'
+            video_id = video_id.rpartition('ref:')[2]
 
             if video_id and account_id and player_id and embed:
                 entries.append(

From 99a9ff24230646d1ec2c93d51942cdf314807f3f Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Tue, 7 Mar 2017 01:02:02 -0500
Subject: [PATCH 12/22] [brightcove:new] ref[3] was deadlink

---
 youtube_dl/extractor/brightcove.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 2cb1bca8c..610ef02b5 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -482,7 +482,7 @@ class BrightcoveNewIE(InfoExtractor):
         # Reference:
         # 1. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideoiniframe
         # 2. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideousingjavascript
-        # 3. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/embed-in-page.html 
+        # 3. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/in-page-embed-player-implementation.html
         # 4. https://support.brightcove.com/en/video-cloud/docs/dynamically-assigning-videos-player
         # 5. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#tag
 

From 15859c6845b158d80e6e3db94198d57602ef333c Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Tue, 7 Mar 2017 01:06:19 -0500
Subject: [PATCH 13/22] [Brightcove:new] reorder doc references in order of
 HTML page

Otherwise it is very confusing to read them out of order.
---
 youtube_dl/extractor/brightcove.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 610ef02b5..d625b802f 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -480,23 +480,23 @@ class BrightcoveNewIE(InfoExtractor):
     @staticmethod
     def _extract_urls(webpage):
         # Reference:
-        # 1. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideoiniframe
+        # 1. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#tag
         # 2. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideousingjavascript
-        # 3. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/in-page-embed-player-implementation.html
-        # 4. https://support.brightcove.com/en/video-cloud/docs/dynamically-assigning-videos-player
-        # 5. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#tag
+        # 3. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideoiniframe
+        # 4. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/in-page-embed-player-implementation.html
+        # 5. https://support.brightcove.com/en/video-cloud/docs/dynamically-assigning-videos-player
 
-        # [5] looks like:
+        # [1] looks like:
         # 	<video data-video-id="5320421710001" data-account="245991542" data-player="SJWAiyYWg" data-embed="default" class="video-js" controls itemscope itemtype="http://schema.org/VideoObject">
 
         entries = []
 
-        # Look for iframe embeds [1]
+        # Look for iframe embeds [3]
         for _, url in re.findall(
                 r'<iframe[^>]+src=(["\'])((?:https?:)?//players\.brightcove\.net/\d+/[^/]+/index\.html.+?)\1', webpage):
             entries.append(url if url.startswith('http') else 'http:' + url)
 
-        # Look for embed_in_page embeds [2] and <video> tags [5]
+        # Look for <video> tags [1] and embed_in_page embeds [2]
         for video, script_tag, account_id, player_id, embed in re.findall(
         r'''(?isx)
             (<video[^>]+>)
@@ -509,7 +509,7 @@ class BrightcoveNewIE(InfoExtractor):
         ''', webpage):
             attrs = extract_attributes(video)
 
-            # According to examples from [3] it's unclear whether video id
+            # According to examples from [4] it's unclear whether video id
             # may be optional and what to do when it is
             video_id    = attrs.get('data-video-id')
             # See PR#12099/bostonglobe.py for 'data-brightcove-video-id' variant
@@ -518,7 +518,7 @@ class BrightcoveNewIE(InfoExtractor):
             if not player_id:  player_id   = attrs.get('data-player')
             if not embed:      embed       = attrs.get('data-embed')
 
-            # According to [4] data-video-id may be prefixed with 'ref:'
+            # According to [5] data-video-id may be prefixed with 'ref:'
             video_id = video_id.rpartition('ref:')[2]
 
             if video_id and account_id and player_id and embed:

From 99474c65105ecaf2a24ced17c35c3b475e07e925 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Tue, 7 Mar 2017 01:50:18 -0500
Subject: [PATCH 14/22] [brightcove:new] can't .rpartition video_id if it's
 None

---
 youtube_dl/extractor/brightcove.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index d625b802f..9f63ab79c 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -519,7 +519,7 @@ class BrightcoveNewIE(InfoExtractor):
             if not embed:      embed       = attrs.get('data-embed')
 
             # According to [5] data-video-id may be prefixed with 'ref:'
-            video_id = video_id.rpartition('ref:')[2]
+            if video_id: video_id = video_id.rpartition('ref:')[2]
 
             if video_id and account_id and player_id and embed:
                 entries.append(

From 63108b883593fe427a93c545f6ea6fb7060103e2 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Tue, 7 Mar 2017 07:52:08 -0500
Subject: [PATCH 15/22] [brightcove:new] flake8 for PEP 8

I really reget not being able to use this compact syntax, as well as to align
with spaces:
            if not account_id: account_id  = attrs.get('data-account')
            if not player_id:  player_id   = attrs.get('data-player')
            if not embed:      embed       = attrs.get('data-embed')
but it really upsets flake8:

brightcove.py:519:30: E701 multiple statements on one line (colon)
brightcove.py:519:42: E221 multiple spaces before operator
brightcove.py:520:29: E701 multiple statements on one line (colon)
brightcove.py:520:30: E241 multiple spaces after ':'
brightcove.py:520:41: E221 multiple spaces before operator
brightcove.py:521:25: E701 multiple statements on one line (colon)
brightcove.py:521:26: E241 multiple spaces after ':'
brightcove.py:521:37: E221 multiple spaces before operator
---
 youtube_dl/extractor/brightcove.py | 35 +++++++++++++++++-------------
 1 file changed, 20 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 9f63ab79c..31a947f4c 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -487,7 +487,7 @@ class BrightcoveNewIE(InfoExtractor):
         # 5. https://support.brightcove.com/en/video-cloud/docs/dynamically-assigning-videos-player
 
         # [1] looks like:
-        # 	<video data-video-id="5320421710001" data-account="245991542" data-player="SJWAiyYWg" data-embed="default" class="video-js" controls itemscope itemtype="http://schema.org/VideoObject">
+        # <video data-video-id="5320421710001" data-account="245991542" data-player="SJWAiyYWg" data-embed="default" class="video-js" controls itemscope itemtype="http://schema.org/VideoObject">
 
         entries = []
 
@@ -498,28 +498,33 @@ class BrightcoveNewIE(InfoExtractor):
 
         # Look for <video> tags [1] and embed_in_page embeds [2]
         for video, script_tag, account_id, player_id, embed in re.findall(
-        r'''(?isx)
-            (<video[^>]+>)
-            (?:.*?
-              (<script[^>]+
-                src=["\'](?:https?:)?//players\.brightcove\.net/
-                (\d+)/([^/]+)_([^/]+)/index(?:\.min)?\.js
-              )
-            )?
-        ''', webpage):
+            r'''(?isx)
+                (<video[^>]+>)
+                (?:.*?
+                  (<script[^>]+
+                    src=["\'](?:https?:)?//players\.brightcove\.net/
+                    (\d+)/([^/]+)_([^/]+)/index(?:\.min)?\.js
+                  )
+                )?
+            ''', webpage
+        ):
             attrs = extract_attributes(video)
 
             # According to examples from [4] it's unclear whether video id
             # may be optional and what to do when it is
-            video_id    = attrs.get('data-video-id')
+            video_id = attrs.get('data-video-id')
             # See PR#12099/bostonglobe.py for 'data-brightcove-video-id' variant
 
-            if not account_id: account_id  = attrs.get('data-account')
-            if not player_id:  player_id   = attrs.get('data-player')
-            if not embed:      embed       = attrs.get('data-embed')
+            if not account_id:
+                account_id = attrs.get('data-account')
+            if not player_id:
+                player_id = attrs.get('data-player')
+            if not embed:
+                embed = attrs.get('data-embed')
 
             # According to [5] data-video-id may be prefixed with 'ref:'
-            if video_id: video_id = video_id.rpartition('ref:')[2]
+            if video_id:
+                video_id = video_id.rpartition('ref:')[2]
 
             if video_id and account_id and player_id and embed:
                 entries.append(

From f6f7ec672136f7e44f5a6e59a93773eeb1e6252b Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Tue, 7 Mar 2017 07:53:12 -0500
Subject: [PATCH 16/22] [Generic] indent fix on Test_Generic26

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 91964219f..091ebb717 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -479,7 +479,7 @@ class GenericIE(InfoExtractor):
                     'upload_date': '20170205',
                     'uploader_id': '800000640001',
                 },
-            'only_matching': True,
+                'only_matching': True,
             }],
         },
         # ooyala video

From c8620243d5dce16a2fc0fb9bcc70b8e311ba7727 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Tue, 7 Mar 2017 08:32:56 -0500
Subject: [PATCH 17/22] [BrightcoveLegacy] mark test_BrightcoveLegacy_4 as gone

Black screen in browsers.

youtube-dl reports:
  [brightcove:legacy] ref:event-stream-356: Downloading webpage
  ERROR: brightcove said: This player is not enabled for access without key.
---
 youtube_dl/extractor/brightcove.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 31a947f4c..5499b30e7 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -110,6 +110,7 @@ class BrightcoveLegacyIE(InfoExtractor):
                 'upload_date': '20140827',
                 'uploader_id': '710858724001',
             },
+            'skip': 'Video gone',
         },
         {
             # playlist with 'videoList'

From ca67fc4341ec49e934f74c8f847aacc385dd643a Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Sat, 18 Mar 2017 11:46:16 -0400
Subject: [PATCH 18/22] [brightcove:new] don't strip ref: from video_id

Partial revert of 49571c1c2fc0872d8e8cf341cdeacd57d8885236
which was based on a misreading of the regexp extractor.
---
 youtube_dl/extractor/brightcove.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index dc192645e..5a910d26d 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -533,10 +533,6 @@ class BrightcoveNewIE(InfoExtractor):
             if not embed:
                 embed = attrs.get('data-embed')
 
-            # According to [5] data-video-id may be prefixed with 'ref:'
-            if video_id:
-                video_id = video_id.rpartition('ref:')[2]
-
             if video_id and account_id and player_id and embed:
                 entries.append(
                     'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'

From 36fc7fb07aeff1df0f4b7f9ab5ba9fa9385384c2 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Sat, 18 Mar 2017 11:47:24 -0400
Subject: [PATCH 19/22] [brightcove:new] reorder [1] <video> comment to point
 of use

---
 youtube_dl/extractor/brightcove.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 5a910d26d..f3443304d 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -497,9 +497,6 @@ class BrightcoveNewIE(InfoExtractor):
         # 4. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/in-page-embed-player-implementation.html
         # 5. https://support.brightcove.com/en/video-cloud/docs/dynamically-assigning-videos-player
 
-        # [1] looks like:
-        # <video data-video-id="5320421710001" data-account="245991542" data-player="SJWAiyYWg" data-embed="default" class="video-js" controls itemscope itemtype="http://schema.org/VideoObject">
-
         entries = []
 
         # Look for iframe embeds [3]
@@ -508,6 +505,9 @@ class BrightcoveNewIE(InfoExtractor):
             entries.append(url if url.startswith('http') else 'http:' + url)
 
         # Look for <video> tags [1] and embed_in_page embeds [2]
+        # [1] looks like:
+        # <video data-video-id="5320421710001" data-account="245991542" data-player="SJWAiyYWg" data-embed="default" class="video-js" controls itemscope itemtype="http://schema.org/VideoObject">
+
         for video, script_tag, account_id, player_id, embed in re.findall(
             r'''(?isx)
                 (<video[^>]+>)

From a3915f69fa05b93b346b3e01fd1b882045ca6b65 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Sat, 18 Mar 2017 11:49:06 -0400
Subject: [PATCH 20/22] [generic] remove Brightcove detection messages

Unnecessarily verbose since the extractor itself
will print messages prefixed by IE_NAME.
---
 youtube_dl/extractor/generic.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 808a76e49..42fd33499 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1887,7 +1887,6 @@ class GenericIE(InfoExtractor):
         # Look for Brightcove Legacy Studio embeds
         bc_urls = BrightcoveLegacyIE._extract_brightcove_urls(webpage)
         if bc_urls:
-            self.to_screen('Brightcove (legacy) video detected.')
             entries = [{
                 '_type': 'url',
                 'url': smuggle_url(bc_url, {'Referer': url}),
@@ -1904,7 +1903,6 @@ class GenericIE(InfoExtractor):
         # Look for Brightcove New Studio embeds
         bc_urls = BrightcoveNewIE._extract_urls(webpage)
         if bc_urls:
-            self.to_screen('Brightcove (new) video detected.')
             return _playlist_from_matches(bc_urls, ie='BrightcoveNew')
 
         # Look for ThePlatform embeds

From 936693bf1bd40b7c8efe74ff067a96813910d2b1 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Sat, 18 Mar 2017 12:09:13 -0400
Subject: [PATCH 21/22] [generic] Skip Test_Generic25, it changes frequently

But it was only added 2 weeks ago by @dstftw in
b68a812ea839e44148516a34a15193189e58ba77, so maybe it should be
handled differently? Remove the test? Mark it only_matching? Not sure.
---
 youtube_dl/extractor/generic.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 42fd33499..1ef4d7d54 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -465,6 +465,7 @@ class GenericIE(InfoExtractor):
             'params': {
                 'skip_download': True,  # m3u8 download
             },
+            'skip': 'video rotates...weekly?',
         },
         {
             # Brightcove:new type [2].

From fe6eb793c07d7b798b08869dce78ee32e991cc48 Mon Sep 17 00:00:00 2001
From: John Hawkinson <jhawk@mit.edu>
Date: Sat, 18 Mar 2017 12:36:45 -0400
Subject: [PATCH 22/22] [Brightcove:new] reorder doc refs in code order (again)

---
 youtube_dl/extractor/brightcove.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index f3443304d..c7059a12a 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -491,21 +491,21 @@ class BrightcoveNewIE(InfoExtractor):
     @staticmethod
     def _extract_urls(webpage):
         # Reference:
-        # 1. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#tag
-        # 2. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideousingjavascript
-        # 3. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideoiniframe
+        # 1. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideoiniframe
+        # 2. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#tag
+        # 3. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideousingjavascript
         # 4. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/in-page-embed-player-implementation.html
         # 5. https://support.brightcove.com/en/video-cloud/docs/dynamically-assigning-videos-player
 
         entries = []
 
-        # Look for iframe embeds [3]
+        # Look for iframe embeds [1]
         for _, url in re.findall(
                 r'<iframe[^>]+src=(["\'])((?:https?:)?//players\.brightcove\.net/\d+/[^/]+/index\.html.+?)\1', webpage):
             entries.append(url if url.startswith('http') else 'http:' + url)
 
-        # Look for <video> tags [1] and embed_in_page embeds [2]
-        # [1] looks like:
+        # Look for <video> tags [2] and embed_in_page embeds [3]
+        # [2] looks like:
         # <video data-video-id="5320421710001" data-account="245991542" data-player="SJWAiyYWg" data-embed="default" class="video-js" controls itemscope itemtype="http://schema.org/VideoObject">
 
         for video, script_tag, account_id, player_id, embed in re.findall(