tonym/13ft
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Apply all review comments from PR #43

Browse Source 
Agent-Logs-Url: https://github.com/wasi-master/13ft/sessions/62de0c61-bb98-4941-9db2-3e6b22b08033

Co-authored-by: wasi-master <63045920+wasi-master@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-05-11 18:28:53 +00:00
committed by GitHub
co-authored by wasi-master
parent e1dd3925b2
commit fc9559ad8b
1 changed files with 67 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/portable.py
+30 -31
View File
@@ -1,15 +1,13 @@
import os
from urllib.parse import urljoin, urlparse
import flask
import re
import requests
import uuid
import threading
import time
from urllib.parse import urljoin, urlparse, quote
from flask import request, Response
from bs4 import BeautifulSoup
from urllib.parse import urlparse, urljoin, quote
app = flask.Flask(__name__)
googlebot_headers = {
@@ -17,6 +15,7 @@ googlebot_headers = {
}
jobs = {}
jobs_lock = threading.Lock()
html = """
<!DOCTYPE html>
@@ -340,9 +339,19 @@ html = """
function showError(message) {
const area = document.getElementById('error-area');
area.innerHTML = '<div class="error-box"><strong>Something went wrong</strong>' +
message + '</div>' +
'<a href="/" class="retry-btn">Try Another URL</a>';
area.innerHTML = '';
const box = document.createElement('div');
box.className = 'error-box';
const strong = document.createElement('strong');
strong.textContent = 'Something went wrong';
box.appendChild(strong);
box.appendChild(document.createTextNode(message));
area.appendChild(box);
const retry = document.createElement('a');
retry.href = '/';
retry.className = 'retry-btn';
retry.textContent = 'Try Another URL';
area.appendChild(retry);
}
document.getElementById('url-form').addEventListener('submit', function(e) {
@@ -360,17 +369,12 @@ html = """
elapsedEl.textContent = sec + 's elapsed';
}, 500);
const formData = new FormData();
formData.append('link', link);
const evtSource = new EventSource('/status/' + encodeURIComponent(link));
let currentStep = 0;
evtSource.addEventListener('step', function(e) {
const data = JSON.parse(e.data);
const idx = STEPS.findIndex(s => s.id === data.step);
if (idx >= 0) {
currentStep = idx;
renderSteps(idx);
const pct = Math.min(((idx + 1) / STEPS.length) * 100, 95);
progressEl.style.width = pct + '%';
@@ -507,9 +511,9 @@ def fetch_via_freedium(url, job_id=None):
def fetch_via_archive_org(url, job_id=None):
set_step(job_id, 'fallback_org')
wayback_api = f"https://archive.org/wayback/available?url={url}"
wayback_api = "https://archive.org/wayback/available"
try:
meta = requests.get(wayback_api, timeout=15).json()
meta = requests.get(wayback_api, params={"url": url}, timeout=15).json()
snapshot = meta.get("archived_snapshots", {}).get("closest", {})
if not snapshot.get("available"):
return None, None
@@ -517,7 +521,6 @@ def fetch_via_archive_org(url, job_id=None):
if archived_url.startswith("http://"):
archived_url = "https://" + archived_url[len("http://"):]
if "/web/" in archived_url and "id_/" not in archived_url:
archived_url = archived_url.replace("/web/", "/web/", 1)
archived_url = archived_url.replace(
archived_url.split("/web/")[1].split("/")[0],
archived_url.split("/web/")[1].split("/")[0] + "id_",
@@ -535,7 +538,7 @@ def fetch_via_archive_ph(url, job_id=None):
set_step(job_id, 'fallback_ph')
for mirror in ARCHIVE_PH_MIRRORS:
try:
newest_url = f"https://{mirror}/newest/{url}"
newest_url = f"https://{mirror}/newest/{quote(url, safe=':/')}"
resp = requests.get(
newest_url,
headers=REAL_BROWSER_HEADERS,
@@ -593,20 +596,6 @@ def bypass_paywall(url, job_id=None):
if not html_text or is_challenge_page(html_text):
recovered = False
if not medium:
archived_html, archived_url = fetch_via_archive_org(url, job_id)
if archived_html and not is_challenge_page(archived_html):
html_text = archived_html
final_url = archived_url
recovered = True
if not recovered:
archived_html, archived_url = fetch_via_archive_ph(url, job_id)
if archived_html and not is_challenge_page(archived_html):
html_text = archived_html
final_url = archived_url
recovered = True
else:
archived_html, archived_url = fetch_via_archive_org(url, job_id)
if archived_html and not is_challenge_page(archived_html):
html_text = archived_html
@@ -645,21 +634,27 @@ def bypass_paywall(url, job_id=None):
def fetch_worker(job_id, url):
try:
result = bypass_paywall(url, job_id)
with jobs_lock:
jobs[job_id]['result'] = result
jobs[job_id]['step'] = 'done'
except requests.exceptions.Timeout:
with jobs_lock:
jobs[job_id]['error'] = 'The website took too long to respond (30s timeout). Try again later.'
jobs[job_id]['step'] = 'error'
except requests.exceptions.ConnectionError:
with jobs_lock:
jobs[job_id]['error'] = 'Could not connect to the website. Check the URL and try again.'
jobs[job_id]['step'] = 'error'
except requests.exceptions.RequestException as e:
with jobs_lock:
jobs[job_id]['error'] = f'Failed to fetch the page: {e}'
jobs[job_id]['step'] = 'error'
except RuntimeError as e:
with jobs_lock:
jobs[job_id]['error'] = str(e)
jobs[job_id]['step'] = 'error'
except Exception as e:
with jobs_lock:
jobs[job_id]['error'] = f'Unexpected error: {e}'
jobs[job_id]['step'] = 'error'
@@ -674,6 +669,7 @@ def status_stream(url):
import json
job_id = str(uuid.uuid4())
with jobs_lock:
jobs[job_id] = {'step': 'queued', 'result': None, 'error': None}
thread = threading.Thread(target=fetch_worker, args=(job_id, url))
@@ -682,8 +678,10 @@ def status_stream(url):
def generate():
last_step = None
try:
while True:
job = jobs.get(job_id)
with jobs_lock:
job = dict(jobs[job_id]) if job_id in jobs else None
if not job:
break
@@ -701,7 +699,8 @@ def status_stream(url):
yield f"event: step\ndata: {json.dumps({'step': current})}\n\n"
time.sleep(0.2)
finally:
with jobs_lock:
jobs.pop(job_id, None)
return Response(generate(), mimetype='text/event-stream',