tonym/13ft
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Format files

Browse Source
This commit is contained in:
dablenparty
2025-10-01 11:34:07 -04:00
parent 52a8f8b32f
commit 71bfa80778
2 changed files with 28 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/index.py
+14 -11
View File
@@ -1,36 +1,39 @@
from urllib.parse import urljoin, urlparse
import flask
import requests
from flask import request
from bs4 import BeautifulSoup
from urllib.parse import urlparse, urljoin
from flask import request
app = flask.Flask(__name__)
googlebot_headers = {
"User-Agent": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.6533.119 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
}
def add_base_tag(html_content, original_url):
soup = BeautifulSoup(html_content, 'html.parser')
soup = BeautifulSoup(html_content, "html.parser")
parsed_url = urlparse(original_url)
base_url = f"{parsed_url.scheme}://{parsed_url.netloc}/"
# Handle paths that are not root, e.g., "https://x.com/some/path/w.html"
if parsed_url.path and not parsed_url.path.endswith('/'):
base_url = urljoin(base_url, parsed_url.path.rsplit('/', 1)[0] + '/')
base_tag = soup.find('base')
if parsed_url.path and not parsed_url.path.endswith("/"):
base_url = urljoin(base_url, parsed_url.path.rsplit("/", 1)[0] + "/")
base_tag = soup.find("base")
print(base_url)
if not base_tag:
new_base_tag = soup.new_tag('base', href=base_url)
new_base_tag = soup.new_tag("base", href=base_url)
if soup.head:
soup.head.insert(0, new_base_tag)
else:
head_tag = soup.new_tag('head')
head_tag = soup.new_tag("head")
head_tag.insert(0, new_base_tag)
soup.insert(0, head_tag)
return str(soup)
def bypass_paywall(url):
"""
Bypass paywall for a given url
app/portable.py
+14 -11
View File
@@ -1,8 +1,9 @@
from urllib.parse import urljoin, urlparse
import flask
import requests
from flask import request
from bs4 import BeautifulSoup
from urllib.parse import urlparse, urljoin
from flask import request
app = flask.Flask(__name__)
googlebot_headers = {
@@ -187,28 +188,30 @@ html = """
</html>
"""
def add_base_tag(html_content, original_url):
soup = BeautifulSoup(html_content, 'html.parser')
soup = BeautifulSoup(html_content, "html.parser")
parsed_url = urlparse(original_url)
base_url = f"{parsed_url.scheme}://{parsed_url.netloc}/"
# Handle paths that are not root, e.g., "https://x.com/some/path/w.html"
if parsed_url.path and not parsed_url.path.endswith('/'):
base_url = urljoin(base_url, parsed_url.path.rsplit('/', 1)[0] + '/')
base_tag = soup.find('base')
if parsed_url.path and not parsed_url.path.endswith("/"):
base_url = urljoin(base_url, parsed_url.path.rsplit("/", 1)[0] + "/")
base_tag = soup.find("base")
print(base_url)
if not base_tag:
new_base_tag = soup.new_tag('base', href=base_url)
new_base_tag = soup.new_tag("base", href=base_url)
if soup.head:
soup.head.insert(0, new_base_tag)
else:
head_tag = soup.new_tag('head')
head_tag = soup.new_tag("head")
head_tag.insert(0, new_base_tag)
soup.insert(0, head_tag)
return str(soup)
def bypass_paywall(url):
"""
Bypass paywall for a given url