incomming mail processing
This commit is contained in:
Olaf
@@ -18,6 +18,7 @@ from backend.app.services.email_service import (
|
||||
send_verification_email,
|
||||
smtp_configured,
|
||||
)
|
||||
from backend.app.services.inbound_email_service import get_imap_settings, imap_configured, save_imap_settings
|
||||
from backend.app.services.email_verification import create_verification_token
|
||||
from backend.app.services.theme_service import THEMES, get_enabled_themes, save_enabled_themes
|
||||
from backend.app.services.secret_store import encrypt_secret
|
||||
@@ -63,6 +64,16 @@ class AdminThemesUpdate(BaseModel):
|
||||
themes: list[str]
|
||||
|
||||
|
||||
class AdminImapUpdate(BaseModel):
|
||||
imap_host: str
|
||||
imap_port: int = 993
|
||||
imap_username: str = ''
|
||||
imap_password: str = ''
|
||||
imap_mailbox: str = 'INBOX'
|
||||
imap_use_ssl: bool = True
|
||||
imap_poll_interval_seconds: int = 60
|
||||
|
||||
|
||||
def validate_smtp_values(payload: AdminSmtpUpdate, current: dict | None = None) -> dict:
|
||||
smtp_host = payload.smtp_host.strip()
|
||||
smtp_from = payload.smtp_from.strip()
|
||||
@@ -80,6 +91,27 @@ def validate_smtp_values(payload: AdminSmtpUpdate, current: dict | None = None)
|
||||
}
|
||||
|
||||
|
||||
def validate_imap_values(payload: AdminImapUpdate, current: dict | None = None) -> dict:
|
||||
imap_host = payload.imap_host.strip()
|
||||
imap_username = payload.imap_username.strip()
|
||||
imap_mailbox = payload.imap_mailbox.strip() or 'INBOX'
|
||||
if not imap_host or not imap_username or not imap_mailbox:
|
||||
raise HTTPException(status_code=422, detail='IMAP host, username, and mailbox are required')
|
||||
if not 1 <= payload.imap_port <= 65535:
|
||||
raise HTTPException(status_code=422, detail='IMAP port must be between 1 and 65535')
|
||||
if payload.imap_poll_interval_seconds < 5:
|
||||
raise HTTPException(status_code=422, detail='IMAP poll interval must be at least 5 seconds')
|
||||
return {
|
||||
'imap_host': imap_host,
|
||||
'imap_port': payload.imap_port,
|
||||
'imap_username': imap_username,
|
||||
'imap_password': payload.imap_password or (current or {}).get('imap_password', ''),
|
||||
'imap_mailbox': imap_mailbox,
|
||||
'imap_use_ssl': payload.imap_use_ssl,
|
||||
'imap_poll_interval_seconds': payload.imap_poll_interval_seconds,
|
||||
}
|
||||
|
||||
|
||||
def public_user(row):
|
||||
return {
|
||||
'id': row['id'],
|
||||
@@ -166,11 +198,29 @@ def public_smtp_settings(values: dict) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def public_imap_settings(values: dict) -> dict:
|
||||
return {
|
||||
'imap_host': values['imap_host'],
|
||||
'imap_port': values['imap_port'],
|
||||
'imap_username': values['imap_username'],
|
||||
'imap_mailbox': values['imap_mailbox'],
|
||||
'imap_use_ssl': values['imap_use_ssl'],
|
||||
'imap_poll_interval_seconds': values['imap_poll_interval_seconds'],
|
||||
'password_configured': bool(values['imap_password']),
|
||||
'configured': imap_configured(values),
|
||||
}
|
||||
|
||||
|
||||
@router.get('/smtp')
|
||||
def get_admin_smtp_settings(_: dict = Depends(require_admin)):
|
||||
return public_smtp_settings(get_smtp_settings())
|
||||
|
||||
|
||||
@router.get('/imap')
|
||||
def get_admin_imap_settings(_: dict = Depends(require_admin)):
|
||||
return public_imap_settings(get_imap_settings())
|
||||
|
||||
|
||||
@router.get('/themes')
|
||||
def get_admin_themes(_: dict = Depends(require_admin)):
|
||||
return {'themes': THEMES, 'enabled': get_enabled_themes()}
|
||||
@@ -195,6 +245,22 @@ def update_admin_smtp_settings(payload: AdminSmtpUpdate, current_user: dict = De
|
||||
return public_smtp_settings(values)
|
||||
|
||||
|
||||
@router.put('/imap')
|
||||
def update_admin_imap_settings(payload: AdminImapUpdate, current_user: dict = Depends(require_admin)):
|
||||
current = get_imap_settings()
|
||||
values = validate_imap_values(payload, current)
|
||||
save_imap_settings(values)
|
||||
record_audit_event(current_user['id'], 'imap_settings_updated', 'application', details={
|
||||
'host': values['imap_host'],
|
||||
'port': values['imap_port'],
|
||||
'username': values['imap_username'],
|
||||
'mailbox': values['imap_mailbox'],
|
||||
'ssl': values['imap_use_ssl'],
|
||||
'poll_interval_seconds': values['imap_poll_interval_seconds'],
|
||||
})
|
||||
return public_imap_settings(values)
|
||||
|
||||
|
||||
@router.post('/smtp/test')
|
||||
def validate_admin_smtp(payload: AdminSmtpUpdate, request: Request, current_user: dict = Depends(require_admin)):
|
||||
values = validate_smtp_values(payload, get_smtp_settings())
|
||||
|
||||
@@ -47,6 +47,13 @@ class Settings:
|
||||
smtp_password: str = os.getenv('LINKLOG_SMTP_PASSWORD', '')
|
||||
smtp_from: str = os.getenv('LINKLOG_SMTP_FROM', 'LinkLog <no-reply@localhost>')
|
||||
smtp_use_tls: bool = os.getenv('LINKLOG_SMTP_USE_TLS', 'true').lower() in {'1', 'true', 'yes'}
|
||||
imap_host: str = os.getenv('LINKLOG_IMAP_HOST', '')
|
||||
imap_port: int = int(os.getenv('LINKLOG_IMAP_PORT', '993'))
|
||||
imap_username: str = os.getenv('LINKLOG_IMAP_USERNAME', '')
|
||||
imap_password: str = os.getenv('LINKLOG_IMAP_PASSWORD', '')
|
||||
imap_mailbox: str = os.getenv('LINKLOG_IMAP_MAILBOX', 'INBOX')
|
||||
imap_use_ssl: bool = os.getenv('LINKLOG_IMAP_USE_SSL', 'true').lower() in {'1', 'true', 'yes'}
|
||||
imap_poll_interval_seconds: int = int(os.getenv('LINKLOG_IMAP_POLL_INTERVAL_SECONDS', '60'))
|
||||
email_verification_expiry_hours: int = int(os.getenv('LINKLOG_EMAIL_VERIFICATION_EXPIRY_HOURS', '24'))
|
||||
password_reset_expiry_hours: int = int(os.getenv('LINKLOG_PASSWORD_RESET_EXPIRY_HOURS', '1'))
|
||||
mastodon_client_name: str = os.getenv('LINKLOG_MASTODON_CLIENT_NAME', 'LinkLog')
|
||||
|
||||
@@ -259,6 +259,22 @@ CREATE TABLE IF NOT EXISTS security_audit_events (
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_security_audit_events_created_at ON security_audit_events(created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_security_audit_events_actor_id ON security_audit_events(actor_id);
|
||||
'''),
|
||||
(18, '''
|
||||
CREATE TABLE IF NOT EXISTS inbound_email_messages (
|
||||
mailbox TEXT NOT NULL,
|
||||
uid TEXT NOT NULL,
|
||||
message_id TEXT,
|
||||
user_id TEXT,
|
||||
link_id TEXT,
|
||||
status TEXT NOT NULL,
|
||||
details TEXT NOT NULL DEFAULT '{}',
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (mailbox, uid),
|
||||
FOREIGN KEY(user_id) REFERENCES users(id) ON DELETE SET NULL,
|
||||
FOREIGN KEY(link_id) REFERENCES links(id) ON DELETE SET NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_inbound_email_messages_message_id ON inbound_email_messages(message_id);
|
||||
''')
|
||||
]
|
||||
|
||||
|
||||
+22
-1
@@ -1,8 +1,10 @@
|
||||
## Copyright © 2026 Olaf Kolkman
|
||||
## SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
from contextlib import asynccontextmanager
|
||||
from fastapi import FastAPI, Request
|
||||
import logging
|
||||
from threading import Event, Thread
|
||||
from uuid import uuid4
|
||||
from fastapi.responses import HTMLResponse
|
||||
from fastapi.responses import RedirectResponse
|
||||
@@ -20,12 +22,31 @@ from backend.app.api.setup import has_administrator
|
||||
from backend.app.api.user_config import router as user_config_router
|
||||
from backend.app.core.config import settings, validate_configuration
|
||||
from backend.app.database import AVATARS_DIR
|
||||
from backend.app.services.inbound_email_service import run_imap_polling
|
||||
from backend.app.services.link_service import get_public_profile
|
||||
|
||||
logging.basicConfig(level=getattr(logging, settings.log_level, logging.INFO))
|
||||
validate_configuration(settings)
|
||||
|
||||
app = FastAPI(title='LinkLog API', version=settings.version)
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_: FastAPI):
|
||||
stop_event = Event()
|
||||
worker = Thread(
|
||||
target=run_imap_polling,
|
||||
args=(stop_event, logging.getLogger('backend.app.services.inbound_email_service')),
|
||||
name='linklog-imap-poller',
|
||||
daemon=True,
|
||||
)
|
||||
worker.start()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
stop_event.set()
|
||||
worker.join(timeout=2)
|
||||
|
||||
|
||||
app = FastAPI(title='LinkLog API', version=settings.version, lifespan=lifespan)
|
||||
|
||||
|
||||
@app.middleware('http')
|
||||
|
||||
@@ -0,0 +1,308 @@
|
||||
## Copyright © 2026 Olaf Kolkman
|
||||
## SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
from email import policy
|
||||
from email.header import decode_header, make_header
|
||||
from email.parser import BytesParser
|
||||
from email.utils import getaddresses
|
||||
from html.parser import HTMLParser
|
||||
import imaplib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from threading import Event
|
||||
|
||||
from backend.app.core.config import settings
|
||||
from backend.app.database import get_connection
|
||||
from backend.app.services.link_service import create_link, normalize_tags
|
||||
from backend.app.services.secret_store import decrypt_secret, encrypt_secret
|
||||
|
||||
URL_PATTERN = re.compile(r'https?://[^\s<>()"\']+')
|
||||
HASHTAG_PATTERN = re.compile(r'(?<!\w)#([A-Za-z0-9][\w-]*)')
|
||||
|
||||
|
||||
class _HTMLTextExtractor(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.text_parts: list[str] = []
|
||||
self.hrefs: list[str] = []
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if data:
|
||||
self.text_parts.append(data)
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
if tag.lower() != 'a':
|
||||
return
|
||||
for name, value in attrs:
|
||||
if name.lower() == 'href' and value:
|
||||
self.hrefs.append(value)
|
||||
|
||||
|
||||
def _decode_header_value(value: str | None) -> str:
|
||||
if not value:
|
||||
return ''
|
||||
return str(make_header(decode_header(value))).strip()
|
||||
|
||||
|
||||
def _extract_message_text(message) -> tuple[str, list[str]]:
|
||||
plain_parts: list[str] = []
|
||||
html_parts: list[str] = []
|
||||
html_hrefs: list[str] = []
|
||||
|
||||
parts = message.walk() if message.is_multipart() else [message]
|
||||
for part in parts:
|
||||
if part.get_content_maintype() == 'multipart':
|
||||
continue
|
||||
content_type = part.get_content_type()
|
||||
try:
|
||||
content = part.get_content()
|
||||
except (LookupError, UnicodeDecodeError):
|
||||
continue
|
||||
if not isinstance(content, str):
|
||||
continue
|
||||
if content_type == 'text/plain':
|
||||
plain_parts.append(content)
|
||||
elif content_type == 'text/html':
|
||||
parser = _HTMLTextExtractor()
|
||||
parser.feed(content)
|
||||
html_parts.append(' '.join(parser.text_parts))
|
||||
html_hrefs.extend(parser.hrefs)
|
||||
|
||||
text = '\n\n'.join(part.strip() for part in (plain_parts or html_parts) if part.strip())
|
||||
urls = list(dict.fromkeys([*html_hrefs, *URL_PATTERN.findall(text)]))
|
||||
return text, urls
|
||||
|
||||
|
||||
def _extract_comment(text: str) -> str:
|
||||
without_links = URL_PATTERN.sub(' ', text)
|
||||
without_tags = HASHTAG_PATTERN.sub(' ', without_links)
|
||||
lines = []
|
||||
for raw_line in without_tags.splitlines():
|
||||
line = ' '.join(raw_line.split()).strip()
|
||||
if line:
|
||||
lines.append(line)
|
||||
return '\n'.join(lines)
|
||||
|
||||
|
||||
def _extract_tags(*values: str) -> list[str]:
|
||||
return normalize_tags([match.group(0) for value in values for match in HASHTAG_PATTERN.finditer(value or '')])
|
||||
|
||||
|
||||
def _resolve_sender_email(message) -> str:
|
||||
addresses = getaddresses(message.get_all('from', []))
|
||||
for _, email in addresses:
|
||||
if email:
|
||||
return email.strip().lower()
|
||||
return ''
|
||||
|
||||
|
||||
def parse_incoming_email(raw_message: bytes) -> dict:
|
||||
message = BytesParser(policy=policy.default).parsebytes(raw_message)
|
||||
subject = _decode_header_value(message.get('Subject'))
|
||||
text, urls = _extract_message_text(message)
|
||||
subject_without_tags = HASHTAG_PATTERN.sub(' ', subject)
|
||||
subject_without_link = URL_PATTERN.sub(' ', subject_without_tags)
|
||||
title = ' '.join(subject_without_link.split()).strip()
|
||||
tags = _extract_tags(subject, text)
|
||||
return {
|
||||
'message_id': _decode_header_value(message.get('Message-ID')),
|
||||
'from_email': _resolve_sender_email(message),
|
||||
'subject': subject,
|
||||
'title': title,
|
||||
'comment': _extract_comment(text),
|
||||
'tags': tags,
|
||||
'url': urls[0] if urls else None,
|
||||
}
|
||||
|
||||
|
||||
def get_imap_settings() -> dict:
|
||||
values = {
|
||||
'imap_host': settings.imap_host,
|
||||
'imap_port': settings.imap_port,
|
||||
'imap_username': settings.imap_username,
|
||||
'imap_password': settings.imap_password,
|
||||
'imap_mailbox': settings.imap_mailbox,
|
||||
'imap_use_ssl': settings.imap_use_ssl,
|
||||
'imap_poll_interval_seconds': settings.imap_poll_interval_seconds,
|
||||
}
|
||||
with get_connection() as conn:
|
||||
row = conn.execute('SELECT value FROM app_settings WHERE name = ?', ('imap',)).fetchone()
|
||||
if row:
|
||||
values.update(json.loads(row['value']))
|
||||
values['imap_password'] = decrypt_secret(values['imap_password'])
|
||||
return values
|
||||
|
||||
|
||||
def save_imap_settings(values: dict) -> None:
|
||||
stored_values = {
|
||||
**values,
|
||||
'imap_password': encrypt_secret(values['imap_password']),
|
||||
}
|
||||
with get_connection() as conn:
|
||||
conn.execute(
|
||||
'''INSERT INTO app_settings (name, value, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(name) DO UPDATE SET value = excluded.value, updated_at = CURRENT_TIMESTAMP''',
|
||||
('imap', json.dumps(stored_values)),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def imap_configured(imap_values: dict | None = None) -> bool:
|
||||
imap = imap_values or get_imap_settings()
|
||||
return bool(imap['imap_host'] and imap['imap_username'] and imap['imap_password'] and imap['imap_mailbox'])
|
||||
|
||||
|
||||
def _find_user_by_email(address: str) -> dict | None:
|
||||
email = address.strip().lower()
|
||||
if not email:
|
||||
return None
|
||||
with get_connection() as conn:
|
||||
row = conn.execute(
|
||||
'SELECT id, username FROM users WHERE lower(email) = ? AND email_verified = 1',
|
||||
(email,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
row = conn.execute(
|
||||
'''SELECT users.id, users.username
|
||||
FROM user_email_addresses
|
||||
JOIN users ON users.id = user_email_addresses.user_id
|
||||
WHERE lower(user_email_addresses.email) = ? AND user_email_addresses.verified = 1''',
|
||||
(email,),
|
||||
).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
|
||||
def _already_processed(mailbox: str, uid: str) -> bool:
|
||||
with get_connection() as conn:
|
||||
row = conn.execute(
|
||||
'SELECT 1 FROM inbound_email_messages WHERE mailbox = ? AND uid = ? LIMIT 1',
|
||||
(mailbox, uid),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
|
||||
def _record_message(mailbox: str, uid: str, status: str, parsed: dict, user_id: str | None = None, link_id: str | None = None, details: dict | None = None) -> None:
|
||||
payload = details or {}
|
||||
with get_connection() as conn:
|
||||
conn.execute(
|
||||
'''INSERT INTO inbound_email_messages (mailbox, uid, message_id, user_id, link_id, status, details)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(mailbox, uid) DO UPDATE SET
|
||||
message_id = excluded.message_id,
|
||||
user_id = excluded.user_id,
|
||||
link_id = excluded.link_id,
|
||||
status = excluded.status,
|
||||
details = excluded.details''',
|
||||
(mailbox, uid, parsed.get('message_id'), user_id, link_id, status, json.dumps(payload)),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def ingest_message(mailbox: str, uid: str, raw_message: bytes) -> dict:
|
||||
if _already_processed(mailbox, uid):
|
||||
return {'status': 'duplicate', 'mailbox': mailbox, 'uid': uid}
|
||||
|
||||
parsed = parse_incoming_email(raw_message)
|
||||
sender = parsed['from_email']
|
||||
if not sender:
|
||||
_record_message(mailbox, uid, 'ignored_missing_sender', parsed)
|
||||
return {'status': 'ignored_missing_sender', 'mailbox': mailbox, 'uid': uid}
|
||||
|
||||
user = _find_user_by_email(sender)
|
||||
if user is None:
|
||||
_record_message(mailbox, uid, 'ignored_unknown_sender', parsed, details={'from_email': sender})
|
||||
return {'status': 'ignored_unknown_sender', 'mailbox': mailbox, 'uid': uid, 'from_email': sender}
|
||||
|
||||
if not parsed['url']:
|
||||
_record_message(mailbox, uid, 'ignored_no_link', parsed, user_id=user['id'])
|
||||
return {'status': 'ignored_no_link', 'mailbox': mailbox, 'uid': uid, 'user_id': user['id']}
|
||||
|
||||
title = parsed['title'] or parsed['url']
|
||||
record = create_link(user['id'], title, parsed['url'], parsed['comment'], None, parsed['tags'])
|
||||
_record_message(mailbox, uid, 'stored', parsed, user_id=user['id'], link_id=record['id'])
|
||||
return {
|
||||
'status': 'stored',
|
||||
'mailbox': mailbox,
|
||||
'uid': uid,
|
||||
'user_id': user['id'],
|
||||
'link_id': record['id'],
|
||||
'link': record,
|
||||
}
|
||||
|
||||
|
||||
def _open_imap_client(imap_values: dict):
|
||||
client_class = imaplib.IMAP4_SSL if imap_values['imap_use_ssl'] else imaplib.IMAP4
|
||||
return client_class(imap_values['imap_host'], imap_values['imap_port'])
|
||||
|
||||
|
||||
def _fetch_message_bytes(response_data) -> bytes | None:
|
||||
for item in response_data or []:
|
||||
if isinstance(item, tuple) and len(item) > 1 and isinstance(item[1], (bytes, bytearray)):
|
||||
return bytes(item[1])
|
||||
return None
|
||||
|
||||
|
||||
def poll_inbox_once(imap_values: dict | None = None, client_factory=None) -> dict:
|
||||
values = imap_values or get_imap_settings()
|
||||
if not imap_configured(values):
|
||||
return {'status': 'not_configured', 'processed': 0, 'stored': 0, 'ignored': 0}
|
||||
|
||||
mailbox = values['imap_mailbox']
|
||||
client = (client_factory or _open_imap_client)(values)
|
||||
processed = 0
|
||||
stored = 0
|
||||
ignored = 0
|
||||
try:
|
||||
client.login(values['imap_username'], values['imap_password'])
|
||||
status, _ = client.select(mailbox)
|
||||
if status != 'OK':
|
||||
raise RuntimeError(f'Could not select IMAP mailbox {mailbox}')
|
||||
status, data = client.uid('search', None, 'UNSEEN')
|
||||
if status != 'OK':
|
||||
raise RuntimeError('Could not list unseen IMAP messages')
|
||||
raw_uids = data[0].split() if data and data[0] else []
|
||||
for uid in raw_uids:
|
||||
uid_value = uid.decode('utf-8') if isinstance(uid, bytes) else str(uid)
|
||||
status, message_data = client.uid('fetch', uid, '(BODY.PEEK[])')
|
||||
if status != 'OK':
|
||||
raise RuntimeError(f'Could not fetch IMAP message {uid_value}')
|
||||
raw_message = _fetch_message_bytes(message_data)
|
||||
if raw_message is None:
|
||||
continue
|
||||
result = ingest_message(mailbox, uid_value, raw_message)
|
||||
processed += 1
|
||||
if result['status'] == 'stored':
|
||||
stored += 1
|
||||
else:
|
||||
ignored += 1
|
||||
if result['status'] in {
|
||||
'stored', 'duplicate', 'ignored_missing_sender', 'ignored_unknown_sender', 'ignored_no_link',
|
||||
}:
|
||||
client.uid('store', uid, '+FLAGS', '(\\Seen)')
|
||||
return {'status': 'ok', 'processed': processed, 'stored': stored, 'ignored': ignored}
|
||||
finally:
|
||||
try:
|
||||
client.logout()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def run_imap_polling(stop_event: Event, logger: logging.Logger | None = None) -> None:
|
||||
active_logger = logger or logging.getLogger(__name__)
|
||||
while not stop_event.is_set():
|
||||
values = get_imap_settings()
|
||||
interval = max(5, int(values.get('imap_poll_interval_seconds', settings.imap_poll_interval_seconds or 60)))
|
||||
if not imap_configured(values):
|
||||
stop_event.wait(interval)
|
||||
continue
|
||||
try:
|
||||
summary = poll_inbox_once(values)
|
||||
if summary['processed']:
|
||||
active_logger.info(
|
||||
'IMAP poll processed=%s stored=%s ignored=%s mailbox=%s',
|
||||
summary['processed'], summary['stored'], summary['ignored'], values['imap_mailbox'],
|
||||
)
|
||||
except Exception as error:
|
||||
active_logger.warning('IMAP polling failed mailbox=%s error=%s', values.get('imap_mailbox'), error)
|
||||
stop_event.wait(interval)
|
||||
Reference in New Issue
Block a user