Markdown from IGF pages

2026-09-18 12:50:59 +02:00
commit 26db01c712
165 changed files with 137619 additions and 0 deletions
@@ -0,0 +1,192 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+import argparse
+import html
+import re
+import unicodedata
+from pathlib import Path
+
+
+TITLE_RE = re.compile(r'<div id="block-pagetitle".*?<h1 class="page-title"><span[^>]*>(.*?)</span>.*?</h1>', re.DOTALL)
+SESSION_RE = re.compile(
+    r'<details[^>]*id="edit-group-session"[^>]*>.*?<div class="details-wrapper">(.*?)</div>\s*</details>',
+    re.DOTALL,
+)
+LABEL_RE = re.compile(r'<div class="field__label">(.*?)</div>', re.DOTALL)
+ITEM_RE = re.compile(r'<div class="field__item">(.*?)</div>', re.DOTALL)
+TAG_RE = re.compile(r'<[^>]+>')
+COMMENT_RE = re.compile(r'<!--.*?-->', re.DOTALL)
+BREAK_RE = re.compile(r'<br\s*/?>', re.IGNORECASE)
+PARA_CLOSE_RE = re.compile(r'</p\s*>', re.IGNORECASE)
+LIST_ITEM_OPEN_RE = re.compile(r'<li\b[^>]*>', re.IGNORECASE)
+LIST_ITEM_CLOSE_RE = re.compile(r'</li\s*>', re.IGNORECASE)
+UL_OPEN_RE = re.compile(r'<ul\b[^>]*>|<ol\b[^>]*>', re.IGNORECASE)
+UL_CLOSE_RE = re.compile(r'</ul\s*>|</ol\s*>', re.IGNORECASE)
+URL_RE = re.compile(r'(https?://\S+)')
+FIELD_CLASS_RE = re.compile(r'class="([^"]*\bfield\b[^"]*)"')
+BOLD_LINE_LABEL_RE = re.compile(r'^(?P<label>[A-Za-z][A-Za-z0-9()\- ]{0,40}):\s*(?P<value>.+)$')
+
+
+def strip_tags(value: str) -> str:
+    return html.unescape(TAG_RE.sub('', value))
+
+
+def normalize_text(value: str) -> str:
+    text = value.replace('\r', '')
+    text = COMMENT_RE.sub('', text)
+    text = BREAK_RE.sub('\n', text)
+    text = PARA_CLOSE_RE.sub('\n\n', text)
+    text = LIST_ITEM_OPEN_RE.sub('- ', text)
+    text = LIST_ITEM_CLOSE_RE.sub('\n', text)
+    text = UL_OPEN_RE.sub('\n', text)
+    text = UL_CLOSE_RE.sub('\n', text)
+    text = strip_tags(text)
+    text = text.replace('\xa0', ' ')
+    text = re.sub(r'^---+\s*', '- ', text, flags=re.MULTILINE)
+    text = re.sub(r'[ \t]+\n', '\n', text)
+    text = re.sub(r'\n[ \t]+', '\n', text)
+    text = re.sub(r'\n{3,}', '\n\n', text)
+    return text.strip()
+
+
+def slugify(value: str) -> str:
+    normalized = unicodedata.normalize('NFKD', value)
+    ascii_only = normalized.encode('ascii', 'ignore').decode('ascii')
+    slug = re.sub(r'[^a-zA-Z0-9]+', '-', ascii_only).strip('-').lower()
+    return slug or 'workshop'
+
+
+def markdown_linkify(text: str) -> str:
+    return URL_RE.sub(r'<\1>', text)
+
+
+def clean_label(value: str) -> str:
+    return normalize_text(value).rstrip(':')
+
+
+def extract_field_blocks(session_html: str) -> list[tuple[str, str]]:
+    blocks: list[tuple[str, str]] = []
+    marker = '<div class="clearfix text-formatted field '
+    index = 0
+
+    while True:
+        start = session_html.find(marker, index)
+        if start == -1:
+            break
+
+        pos = start
+        depth = 0
+        while pos < len(session_html):
+            next_open = session_html.find('<div', pos)
+            next_close = session_html.find('</div>', pos)
+
+            if next_close == -1:
+                raise ValueError('Unbalanced div while parsing session field blocks')
+
+            if next_open != -1 and next_open < next_close:
+                depth += 1
+                pos = next_open + 4
+                continue
+
+            depth -= 1
+            pos = next_close + len('</div>')
+            if depth == 0:
+                break
+
+        block = session_html[start:pos]
+        class_match = FIELD_CLASS_RE.search(block)
+        classes = class_match.group(1) if class_match else ''
+        blocks.append((classes, block))
+        index = pos
+
+    return blocks
+
+
+def format_field(label: str, content: str) -> str:
+    content = content.strip()
+    if not content:
+        return ''
+
+    if label == 'Description' and content.startswith('Description:'):
+        content = content[len('Description:') :].lstrip()
+
+    formatted_lines: list[str] = []
+    for line in content.splitlines():
+        stripped = line.strip()
+        match = BOLD_LINE_LABEL_RE.match(stripped)
+        if match and not stripped.endswith(':'):
+            formatted_lines.append(f"**{match.group('label')}:** {match.group('value')}")
+        else:
+            formatted_lines.append(line)
+
+    content = '\n'.join(formatted_lines)
+    content = markdown_linkify(content)
+    return f"## {label}\n\n{content}"
+
+
+def extract_title(source: str) -> str:
+    title_match = TITLE_RE.search(source)
+    if not title_match:
+        raise ValueError('Could not find title in block-pagetitle')
+    return normalize_text(title_match.group(1))
+
+
+def extract_markdown(source: str) -> tuple[str, str]:
+    title = extract_title(source)
+    session_match = SESSION_RE.search(source)
+    if not session_match:
+        raise ValueError('Could not find session details block by id edit-group-session')
+
+    details_html = session_match.group(1)
+    sections: list[str] = [f'# {title}']
+
+    for classes, body in extract_field_blocks(details_html):
+        label_match = LABEL_RE.search(body)
+        item_match = ITEM_RE.search(body)
+        label = clean_label(label_match.group(1)) if label_match else ''
+
+        if not label:
+            if 'field--name-field-session-content' in classes:
+                label = 'Description'
+            elif 'field--name-field-discussion-facilitation' in classes:
+                label = 'Hybrid Format'
+            elif 'field--name-field-organizers-information' in classes:
+                label = 'Organizers'
+            elif 'field--name-field-speakers' in classes:
+                label = 'Speakers'
+            elif 'field--name-field-reference-document' in classes:
+                label = 'Reference Document'
+            elif 'field--name-field-issues' in classes:
+                label = 'Participant Benefits'
+            else:
+                continue
+
+        content = normalize_text(item_match.group(1) if item_match else body)
+        section = format_field(label, content)
+        if section:
+            sections.append(section)
+
+    return title, '\n\n'.join(sections) + '\n'
+
+
+def convert_file(html_path: Path) -> Path:
+    source = html_path.read_text(encoding='utf-8')
+    title, markdown = extract_markdown(source)
+    markdown_path = html_path.with_name(f'{slugify(title)}.md')
+    markdown_path.write_text(markdown, encoding='utf-8')
+    return markdown_path
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description='Convert IGF workshop HTML files to Markdown.')
+    parser.add_argument('directory', type=Path, help='Directory containing workshop HTML files')
+    args = parser.parse_args()
+
+    for html_file in sorted(args.directory.glob('*.html')):
+        print(convert_file(html_file))
+
+
+if __name__ == '__main__':
+    main()