201 lines
6.8 KiB
Python
201 lines
6.8 KiB
Python
#!/usr/bin/env python3
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import html
|
|
import re
|
|
import unicodedata
|
|
from pathlib import Path
|
|
|
|
|
|
TITLE_RE = re.compile(r'<div id="block-pagetitle".*?<h1 class="page-title"><span[^>]*>(.*?)</span>.*?</h1>', re.DOTALL)
|
|
CANONICAL_RE = re.compile(r'<link rel="canonical" href="([^"]+)"\s*/?>', re.DOTALL)
|
|
SESSION_RE = re.compile(
|
|
r'<details[^>]*id="edit-group-session"[^>]*>.*?<div class="details-wrapper">(.*?)</div>\s*</details>',
|
|
re.DOTALL,
|
|
)
|
|
LABEL_RE = re.compile(r'<div class="field__label">(.*?)</div>', re.DOTALL)
|
|
ITEM_RE = re.compile(r'<div class="field__item">(.*?)</div>', re.DOTALL)
|
|
TAG_RE = re.compile(r'<[^>]+>')
|
|
COMMENT_RE = re.compile(r'<!--.*?-->', re.DOTALL)
|
|
BREAK_RE = re.compile(r'<br\s*/?>', re.IGNORECASE)
|
|
PARA_CLOSE_RE = re.compile(r'</p\s*>', re.IGNORECASE)
|
|
LIST_ITEM_OPEN_RE = re.compile(r'<li\b[^>]*>', re.IGNORECASE)
|
|
LIST_ITEM_CLOSE_RE = re.compile(r'</li\s*>', re.IGNORECASE)
|
|
UL_OPEN_RE = re.compile(r'<ul\b[^>]*>|<ol\b[^>]*>', re.IGNORECASE)
|
|
UL_CLOSE_RE = re.compile(r'</ul\s*>|</ol\s*>', re.IGNORECASE)
|
|
URL_RE = re.compile(r'(https?://\S+)')
|
|
FIELD_CLASS_RE = re.compile(r'class="([^"]*\bfield\b[^"]*)"')
|
|
BOLD_LINE_LABEL_RE = re.compile(r'^(?P<label>[A-Za-z][A-Za-z0-9()\- ]{0,40}):\s*(?P<value>.+)$')
|
|
|
|
|
|
def strip_tags(value: str) -> str:
|
|
return html.unescape(TAG_RE.sub('', value))
|
|
|
|
|
|
def normalize_text(value: str) -> str:
|
|
text = value.replace('\r', '')
|
|
text = COMMENT_RE.sub('', text)
|
|
text = BREAK_RE.sub('\n', text)
|
|
text = PARA_CLOSE_RE.sub('\n\n', text)
|
|
text = LIST_ITEM_OPEN_RE.sub('- ', text)
|
|
text = LIST_ITEM_CLOSE_RE.sub('\n', text)
|
|
text = UL_OPEN_RE.sub('\n', text)
|
|
text = UL_CLOSE_RE.sub('\n', text)
|
|
text = strip_tags(text)
|
|
text = text.replace('\xa0', ' ')
|
|
text = re.sub(r'^---+\s*', '- ', text, flags=re.MULTILINE)
|
|
text = re.sub(r'[ \t]+\n', '\n', text)
|
|
text = re.sub(r'\n[ \t]+', '\n', text)
|
|
text = re.sub(r'\n{3,}', '\n\n', text)
|
|
return text.strip()
|
|
|
|
|
|
def slugify(value: str) -> str:
|
|
normalized = unicodedata.normalize('NFKD', value)
|
|
ascii_only = normalized.encode('ascii', 'ignore').decode('ascii')
|
|
slug = re.sub(r'[^a-zA-Z0-9]+', '-', ascii_only).strip('-').lower()
|
|
return slug or 'workshop'
|
|
|
|
|
|
def markdown_linkify(text: str) -> str:
|
|
return URL_RE.sub(r'<\1>', text)
|
|
|
|
|
|
def clean_label(value: str) -> str:
|
|
return normalize_text(value).rstrip(':')
|
|
|
|
|
|
def extract_field_blocks(session_html: str) -> list[tuple[str, str]]:
|
|
blocks: list[tuple[str, str]] = []
|
|
marker = '<div class="clearfix text-formatted field '
|
|
index = 0
|
|
|
|
while True:
|
|
start = session_html.find(marker, index)
|
|
if start == -1:
|
|
break
|
|
|
|
pos = start
|
|
depth = 0
|
|
while pos < len(session_html):
|
|
next_open = session_html.find('<div', pos)
|
|
next_close = session_html.find('</div>', pos)
|
|
|
|
if next_close == -1:
|
|
raise ValueError('Unbalanced div while parsing session field blocks')
|
|
|
|
if next_open != -1 and next_open < next_close:
|
|
depth += 1
|
|
pos = next_open + 4
|
|
continue
|
|
|
|
depth -= 1
|
|
pos = next_close + len('</div>')
|
|
if depth == 0:
|
|
break
|
|
|
|
block = session_html[start:pos]
|
|
class_match = FIELD_CLASS_RE.search(block)
|
|
classes = class_match.group(1) if class_match else ''
|
|
blocks.append((classes, block))
|
|
index = pos
|
|
|
|
return blocks
|
|
|
|
|
|
def format_field(label: str, content: str) -> str:
|
|
content = content.strip()
|
|
if not content:
|
|
return ''
|
|
|
|
if label == 'Description' and content.startswith('Description:'):
|
|
content = content[len('Description:') :].lstrip()
|
|
|
|
formatted_lines: list[str] = []
|
|
for line in content.splitlines():
|
|
stripped = line.strip()
|
|
match = BOLD_LINE_LABEL_RE.match(stripped)
|
|
if match and not stripped.endswith(':'):
|
|
formatted_lines.append(f"**{match.group('label')}:** {match.group('value')}")
|
|
else:
|
|
formatted_lines.append(line)
|
|
|
|
content = '\n'.join(formatted_lines)
|
|
content = markdown_linkify(content)
|
|
return f"## {label}\n\n{content}"
|
|
|
|
|
|
def extract_title(source: str) -> str:
|
|
title_match = TITLE_RE.search(source)
|
|
if not title_match:
|
|
raise ValueError('Could not find title in block-pagetitle')
|
|
return normalize_text(title_match.group(1))
|
|
|
|
|
|
def extract_canonical_url(source: str) -> str:
|
|
canonical_match = CANONICAL_RE.search(source)
|
|
if not canonical_match:
|
|
raise ValueError('Could not find canonical URL for proposal page')
|
|
return html.unescape(canonical_match.group(1)).strip()
|
|
|
|
|
|
def extract_markdown(source: str) -> tuple[str, str]:
|
|
title = extract_title(source)
|
|
canonical_url = extract_canonical_url(source)
|
|
session_match = SESSION_RE.search(source)
|
|
if not session_match:
|
|
raise ValueError('Could not find session details block by id edit-group-session')
|
|
|
|
details_html = session_match.group(1)
|
|
sections: list[str] = [f'# [{title}]({canonical_url})']
|
|
|
|
for classes, body in extract_field_blocks(details_html):
|
|
label_match = LABEL_RE.search(body)
|
|
item_match = ITEM_RE.search(body)
|
|
label = clean_label(label_match.group(1)) if label_match else ''
|
|
|
|
if not label:
|
|
if 'field--name-field-session-content' in classes:
|
|
label = 'Description'
|
|
elif 'field--name-field-discussion-facilitation' in classes:
|
|
label = 'Hybrid Format'
|
|
elif 'field--name-field-organizers-information' in classes:
|
|
label = 'Organizers'
|
|
elif 'field--name-field-speakers' in classes:
|
|
label = 'Speakers'
|
|
elif 'field--name-field-reference-document' in classes:
|
|
label = 'Reference Document'
|
|
elif 'field--name-field-issues' in classes:
|
|
label = 'Participant Benefits'
|
|
else:
|
|
continue
|
|
|
|
content = normalize_text(item_match.group(1) if item_match else body)
|
|
section = format_field(label, content)
|
|
if section:
|
|
sections.append(section)
|
|
|
|
return title, '\n\n'.join(sections) + '\n'
|
|
|
|
|
|
def convert_file(html_path: Path) -> Path:
|
|
source = html_path.read_text(encoding='utf-8')
|
|
title, markdown = extract_markdown(source)
|
|
markdown_path = html_path.with_name(f'{slugify(title)}.md')
|
|
markdown_path.write_text(markdown, encoding='utf-8')
|
|
return markdown_path
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description='Convert IGF workshop HTML files to Markdown.')
|
|
parser.add_argument('directory', type=Path, help='Directory containing workshop HTML files')
|
|
args = parser.parse_args()
|
|
|
|
for html_file in sorted(args.directory.glob('*.html')):
|
|
print(convert_file(html_file))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main() |