change encoding based on content

This helps users trying to read content with other encodings.
2023-05-31 10:42:21 -03:00
parent 9191b9339f
commit 5c6c30b95f
1 changed files with 1 additions and 0 deletions
@@ -93,6 +93,7 @@ def bypass_paywall(url):
    Bypass paywall for a given url
    """
    response = requests.get(url, headers=googlebot_headers)
+    response.encoding = response.apparent_encoding
    return response.text