New Entry functionality
This commit is contained in:
@@ -11,6 +11,7 @@ from backend.app.database import get_connection
|
||||
from backend.app.services.plugin_manager import plugin_manager
|
||||
from backend.app.services.token_service import validate_token
|
||||
from backend.app.services.audit_service import record_audit_event
|
||||
from backend.app.services.scraper_service import scrape_title
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -36,6 +37,21 @@ def available_tags():
|
||||
return list_tags()
|
||||
|
||||
|
||||
@router.get('/scrape')
|
||||
def scrape_url(url: str, authorization: str | None = Header(default=None)):
|
||||
if not authorization or not authorization.startswith('Bearer '):
|
||||
raise HTTPException(status_code=401, detail='Missing or invalid Authorization header')
|
||||
info = validate_token(authorization.replace('Bearer ', '', 1))
|
||||
if info is None:
|
||||
raise HTTPException(status_code=401, detail='Token expired or invalid')
|
||||
try:
|
||||
title = scrape_title(url)
|
||||
return {'title': title}
|
||||
except Exception as e:
|
||||
logger.error('Scrape error for %s: %s', url, e)
|
||||
raise HTTPException(status_code=502, detail='Could not scrape URL') from e
|
||||
|
||||
|
||||
@router.get('/links/check')
|
||||
def check_existing_link(
|
||||
title: str,
|
||||
|
||||
@@ -78,6 +78,13 @@ async def labels_page(request: Request):
|
||||
return templates.TemplateResponse(request, 'labels.html', {})
|
||||
|
||||
|
||||
@app.get('/new-entry', response_class=HTMLResponse)
|
||||
async def new_entry_page(request: Request):
|
||||
if not has_administrator():
|
||||
return RedirectResponse('/setup')
|
||||
return templates.TemplateResponse(request, 'new-entry.html', {})
|
||||
|
||||
|
||||
@app.get('/about', response_class=HTMLResponse)
|
||||
async def about_page(request: Request):
|
||||
return templates.TemplateResponse(request, 'about.html', {})
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
## Copyright © 2026 Olaf Kolkman
|
||||
## SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import httpx
|
||||
import logging
|
||||
from html.parser import HTMLParser
|
||||
from urllib.parse import urlparse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TitleParser(HTMLParser):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.title = None
|
||||
self.og_title = None
|
||||
self.twitter_title = None
|
||||
self.meta_title = None
|
||||
self.in_title = False
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
if tag.lower() == 'title':
|
||||
self.in_title = True
|
||||
elif tag.lower() == 'meta':
|
||||
attrs_dict = dict(attrs)
|
||||
# Check for Open Graph title
|
||||
if attrs_dict.get('property', '').lower() == 'og:title':
|
||||
content = attrs_dict.get('content', '').strip()
|
||||
if content and not self.og_title:
|
||||
self.og_title = content
|
||||
# Check for Twitter title
|
||||
elif attrs_dict.get('name', '').lower() == 'twitter:title':
|
||||
content = attrs_dict.get('content', '').strip()
|
||||
if content and not self.twitter_title:
|
||||
self.twitter_title = content
|
||||
# Check for generic meta title
|
||||
elif attrs_dict.get('name', '').lower() == 'title':
|
||||
content = attrs_dict.get('content', '').strip()
|
||||
if content and not self.meta_title:
|
||||
self.meta_title = content
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if tag.lower() == 'title':
|
||||
self.in_title = False
|
||||
|
||||
def handle_data(self, data):
|
||||
if self.in_title and not self.title:
|
||||
stripped = data.strip()
|
||||
if stripped:
|
||||
self.title = stripped
|
||||
|
||||
def get_best_title(self):
|
||||
"""Return the best title found, matching browser behavior.
|
||||
Priority: page <title> tag (what browser shows), then meta tags as fallback."""
|
||||
return self.title or self.og_title or self.twitter_title or self.meta_title
|
||||
|
||||
|
||||
def scrape_title(url: str) -> str:
|
||||
"""
|
||||
Scrape the title from a URL, checking multiple sources:
|
||||
1. Page title tag (what browser shows)
|
||||
2. Open Graph title (og:title meta tag)
|
||||
3. Twitter title (twitter:title meta tag)
|
||||
4. Generic meta title
|
||||
5. Domain name as fallback
|
||||
"""
|
||||
try:
|
||||
# Parse URL to extract domain as fallback
|
||||
parsed = urlparse(url)
|
||||
domain = parsed.netloc or url
|
||||
|
||||
# Fetch the URL with a timeout and size limit, using iter_bytes for decompression
|
||||
with httpx.stream('GET', url, follow_redirects=True, timeout=5.0) as response:
|
||||
if response.status_code != 200:
|
||||
logger.warning('Failed to fetch %s: status %d', url, response.status_code)
|
||||
return domain
|
||||
|
||||
# Read HTML in chunks (auto-decompressed) to avoid loading huge files
|
||||
html_content = b''
|
||||
max_size = 1024 * 100 # 100 KB limit
|
||||
for chunk in response.iter_bytes():
|
||||
html_content += chunk
|
||||
if len(html_content) > max_size:
|
||||
break
|
||||
|
||||
# Parse the HTML to extract title
|
||||
try:
|
||||
html_text = html_content.decode('utf-8', errors='ignore')
|
||||
parser = TitleParser()
|
||||
parser.feed(html_text)
|
||||
best_title = parser.get_best_title()
|
||||
if best_title:
|
||||
return best_title
|
||||
except Exception as e:
|
||||
logger.warning('Failed to parse HTML from %s: %s', url, e)
|
||||
|
||||
return domain
|
||||
except httpx.TimeoutException:
|
||||
logger.warning('Timeout fetching %s', url)
|
||||
return urlparse(url).netloc or url
|
||||
except httpx.NetworkError as e:
|
||||
logger.warning('Network error fetching %s: %s', url, e)
|
||||
return urlparse(url).netloc or url
|
||||
except Exception as e:
|
||||
logger.error('Unexpected error scraping %s: %s', url, e)
|
||||
return urlparse(url).netloc or url
|
||||
Reference in New Issue
Block a user