tonym/13ft
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Format files

Browse Source
This commit is contained in:
dablenparty
2025-10-01 11:34:07 -04:00
parent 52a8f8b32f
commit 71bfa80778
2 changed files with 28 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/index.py
+14 -11
View File
@@ -1,36 +1,39 @@
from urllib.parse import urljoin, urlparse
import flask import flask
import requests import requests
from flask import request
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from urllib.parse import urlparse, urljoin from flask import request
app = flask.Flask(__name__) app = flask.Flask(__name__)
googlebot_headers = { googlebot_headers = {
"User-Agent": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.6533.119 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" "User-Agent": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.6533.119 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
} }
def add_base_tag(html_content, original_url): def add_base_tag(html_content, original_url):
soup = BeautifulSoup(html_content, 'html.parser') soup = BeautifulSoup(html_content, "html.parser")
parsed_url = urlparse(original_url) parsed_url = urlparse(original_url)
base_url = f"{parsed_url.scheme}://{parsed_url.netloc}/" base_url = f"{parsed_url.scheme}://{parsed_url.netloc}/"
# Handle paths that are not root, e.g., "https://x.com/some/path/w.html" # Handle paths that are not root, e.g., "https://x.com/some/path/w.html"
if parsed_url.path and not parsed_url.path.endswith('/'): if parsed_url.path and not parsed_url.path.endswith("/"):
base_url = urljoin(base_url, parsed_url.path.rsplit('/', 1)[0] + '/') base_url = urljoin(base_url, parsed_url.path.rsplit("/", 1)[0] + "/")
base_tag = soup.find('base') base_tag = soup.find("base")
print(base_url) print(base_url)
if not base_tag: if not base_tag:
new_base_tag = soup.new_tag('base', href=base_url) new_base_tag = soup.new_tag("base", href=base_url)
if soup.head: if soup.head:
soup.head.insert(0, new_base_tag) soup.head.insert(0, new_base_tag)
else: else:
head_tag = soup.new_tag('head') head_tag = soup.new_tag("head")
head_tag.insert(0, new_base_tag) head_tag.insert(0, new_base_tag)
soup.insert(0, head_tag) soup.insert(0, head_tag)
return str(soup) return str(soup)
def bypass_paywall(url): def bypass_paywall(url):
""" """
Bypass paywall for a given url Bypass paywall for a given url
app/portable.py
+14 -11
View File
@@ -1,8 +1,9 @@
from urllib.parse import urljoin, urlparse
import flask import flask
import requests import requests
from flask import request
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from urllib.parse import urlparse, urljoin from flask import request
app = flask.Flask(__name__) app = flask.Flask(__name__)
googlebot_headers = { googlebot_headers = {
@@ -187,28 +188,30 @@ html = """
</html> </html>
""" """
def add_base_tag(html_content, original_url): def add_base_tag(html_content, original_url):
soup = BeautifulSoup(html_content, 'html.parser') soup = BeautifulSoup(html_content, "html.parser")
parsed_url = urlparse(original_url) parsed_url = urlparse(original_url)
base_url = f"{parsed_url.scheme}://{parsed_url.netloc}/" base_url = f"{parsed_url.scheme}://{parsed_url.netloc}/"
# Handle paths that are not root, e.g., "https://x.com/some/path/w.html" # Handle paths that are not root, e.g., "https://x.com/some/path/w.html"
if parsed_url.path and not parsed_url.path.endswith('/'): if parsed_url.path and not parsed_url.path.endswith("/"):
base_url = urljoin(base_url, parsed_url.path.rsplit('/', 1)[0] + '/') base_url = urljoin(base_url, parsed_url.path.rsplit("/", 1)[0] + "/")
base_tag = soup.find('base') base_tag = soup.find("base")
print(base_url) print(base_url)
if not base_tag: if not base_tag:
new_base_tag = soup.new_tag('base', href=base_url) new_base_tag = soup.new_tag("base", href=base_url)
if soup.head: if soup.head:
soup.head.insert(0, new_base_tag) soup.head.insert(0, new_base_tag)
else: else:
head_tag = soup.new_tag('head') head_tag = soup.new_tag("head")
head_tag.insert(0, new_base_tag) head_tag.insert(0, new_base_tag)
soup.insert(0, head_tag) soup.insert(0, head_tag)
return str(soup) return str(soup)
def bypass_paywall(url): def bypass_paywall(url):
""" """
Bypass paywall for a given url Bypass paywall for a given url