Fix relative links

Closes https://github.com/wasi-master/13ft/issues/10
This commit is contained in:
Wasi Master
2024-08-14 15:40:07 +06:00
parent 1f43c8a866
commit ffefe1193b
2 changed files with 25 additions and 1 deletions
+24 -1
View File
@@ -1,12 +1,35 @@
import flask import flask
import requests import requests
from flask import request from flask import request
from bs4 import BeautifulSoup
from urllib.parse import urlparse, urljoin
app = flask.Flask(__name__) app = flask.Flask(__name__)
googlebot_headers = { googlebot_headers = {
"User-Agent": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" "User-Agent": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
} }
def add_base_tag(html_content, original_url):
soup = BeautifulSoup(html_content, 'html.parser')
parsed_url = urlparse(original_url)
base_url = f"{parsed_url.scheme}://{parsed_url.netloc}/"
# Handle paths that are not root, e.g., "https://x.com/some/path/w.html"
if parsed_url.path and not parsed_url.path.endswith('/'):
base_url = urljoin(base_url, parsed_url.path.rsplit('/', 1)[0] + '/')
base_tag = soup.find('base')
print(base_url)
if not base_tag:
new_base_tag = soup.new_tag('base', href=base_url)
if soup.head:
soup.head.insert(0, new_base_tag)
else:
head_tag = soup.new_tag('head')
head_tag.insert(0, new_base_tag)
soup.insert(0, head_tag)
return str(soup)
def bypass_paywall(url): def bypass_paywall(url):
""" """
@@ -15,7 +38,7 @@ def bypass_paywall(url):
if url.startswith("http"): if url.startswith("http"):
response = requests.get(url, headers=googlebot_headers) response = requests.get(url, headers=googlebot_headers)
response.encoding = response.apparent_encoding response.encoding = response.apparent_encoding
return response.text return add_base_tag(response.text, response.url)
try: try:
return bypass_paywall("https://" + url) return bypass_paywall("https://" + url)
+1
View File
@@ -1,2 +1,3 @@
flask flask
requests requests
bs4