Crawler: better rss bridge twitter enhancement

jaesivsm · Nov 13, 2020 · e7e3ae6 · e7e3ae6
1 parent 92f8b69
commit e7e3ae6
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 0 deletions.
diff --git a/jarr/crawler/article_builders/rss_bridge.py b/jarr/crawler/article_builders/rss_bridge.py
@@ -26,11 +26,14 @@ def enhance(self):
             return
         soup = BeautifulSoup(content, 'html.parser')
         og_link = self.article['link']
+        og_comments = self.article.get('comments')
         try:  # trying to find the last link in the tweet
             last_link = soup.find_all('a')[-1]
+            self.article['comments'] = self.article['link']
             self.article['link'] = last_link.attrs['href']
         except (KeyError, AttributeError, TypeError, IndexError):
             self.article['link'] = og_link
+            self.article['comments'] = og_comments
         else:
             try:  # link is the image if the link contains the images
                 img = last_link.find_all('img')[0]

diff --git a/tests/libs/rss_bridg_inte_test.py b/tests/libs/rss_bridg_inte_test.py
@@ -35,6 +35,7 @@ def test_rss_twitter_bridge_link_handling(self):
         builder.enhance()
         self.assertEqual("https://www.enercoop.fr/content/licoornes-les-cooper"
                          "atives-du-monde-dapres", builder.article['link'])
+        self.assertEqual(entry['link'], builder.article['comments'])
         self.assertIsNone(builder.article.get('article_type'))
 
     def test_rss_twitter_bridge_img_handling(self):
@@ -45,4 +46,5 @@ def test_rss_twitter_bridge_img_handling(self):
         builder.enhance()
         self.assertEqual("https://pbs.twimg.com/media/EmZUUQxXcAAxMTp.jpg",
                          builder.article['link'])
+        self.assertEqual(entry['link'], builder.article['comments'])
         self.assertEqual('image', builder.article['article_type'].value)