height correction, new defalt

tests/test_ahn.py

@@ -0,0 +1,305 @@
+import json
+from pathlib import Path
+from unittest.mock import Mock
+from urllib.parse import parse_qs, urlparse
+from xml.etree import ElementTree as ET
+
+import numpy as np
+import pytest
+import rasterio
+from pyproj import Transformer
+from rasterio.io import MemoryFile
+from rasterio.transform import from_bounds
+from shapely.geometry import box
+
+from flight_map.ahn import SIZE, coverage_url, terrain_data
+from flight_map.geo import DEFAULT_BOUNDS, Bounds
+from flight_map.maps import water_areas
+from flight_map.network import request_bytes
+from flight_map.plot import SVG_NS, render
+from flight_map.storage import (
+    Observation,
+    checkpoint,
+    create_session,
+    database,
+    initialize,
+    save_observations,
+)
+from flight_map.terrain import Elevation
+
+BOUNDS = DEFAULT_BOUNDS
+
+
+def ahn_bytes(value: float = -3) -> bytes:
+    bounds = Transformer.from_crs(4326, 28992, always_xy=True).transform_bounds(*BOUNDS.values)
+    with MemoryFile() as memory:
+        with memory.open(
+            driver="GTiff",
+            width=SIZE,
+            height=SIZE,
+            count=1,
+            dtype="float32",
+            crs="EPSG:28992",
+            transform=from_bounds(*bounds, SIZE, SIZE),
+            nodata=-9999,
+        ) as dataset:
+            dataset.write(np.full((SIZE, SIZE), value, dtype="float32"), 1)
+        return memory.read()
+
+
+def test_pdok_request_cache_refresh_and_metadata(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    fetch = Mock(return_value=ahn_bytes())
+    monkeypatch.setattr("flight_map.ahn.request_bytes", fetch)
+    path = terrain_data(BOUNDS, tmp_path)
+    assert path is not None
+    assert terrain_data(BOUNDS, tmp_path) == path
+    fetch.assert_called_once()
+    params = parse_qs(urlparse(fetch.call_args.args[0]).query)
+    assert params["coverage"] == ["dtm_05m"]
+    assert params["crs"] == ["EPSG:28992"]
+    assert params["format"] == ["GEOTIFF"]
+    assert params["width"] == params["height"] == [str(SIZE)]
+    with rasterio.open(path) as dataset:
+        assert dataset.tags()["vertical_reference"] == "NAP"
+        assert dataset.tags()["license"] == "CC0"
+    assert terrain_data(BOUNDS, tmp_path, refresh=True) == path
+    assert fetch.call_count == 2
+    assert coverage_url(BOUNDS) != coverage_url(Bounds(4.7, 52.2, 4.8, 52.3))
+    before = path.read_bytes()
+    fetch.side_effect = RuntimeError("HTTP 503")
+    assert terrain_data(BOUNDS, tmp_path, refresh=True) is None
+    assert path.read_bytes() == before
+    assert len(list(tmp_path.glob("*.tif"))) == 1
+
+
+def test_unavailable_corrupt_and_outside_coverage_warn(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    fetch = Mock(return_value=b"<ServiceException>no coverage</ServiceException>")
+    monkeypatch.setattr("flight_map.ahn.request_bytes", fetch)
+    assert terrain_data(BOUNDS, tmp_path) is None
+    assert "height map unavailable" in caplog.text
+    assert not list(tmp_path.iterdir())
+    fetch.reset_mock()
+    assert terrain_data(Bounds(-1, 50, 0, 51), tmp_path) is None
+    fetch.assert_not_called()
+    assert "outside Dutch AHN coverage" in caplog.text
+    fetch.return_value = ahn_bytes()
+    path = terrain_data(BOUNDS, tmp_path)
+    assert path is not None
+    path.write_bytes(b"broken")
+    assert terrain_data(BOUNDS, tmp_path) is None
+    assert "refresh-terrain" in caplog.text
+    assert terrain_data(BOUNDS, tmp_path, refresh=True) == path
+
+
+def test_failed_cache_replace_keeps_old_height_map(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr("flight_map.ahn.request_bytes", Mock(return_value=ahn_bytes()))
+    path = terrain_data(BOUNDS, tmp_path)
+    assert path is not None
+    before = path.read_bytes()
+    monkeypatch.setattr("flight_map.ahn.os.replace", Mock(side_effect=OSError("disk error")))
+    assert terrain_data(BOUNDS, tmp_path, refresh=True) is None
+    assert path.read_bytes() == before
+    assert list(tmp_path.iterdir()) == [path]
+
+
+def water_way(
+    points: list[tuple[float, float]],
+    identifier: int = 1,
+) -> dict[str, object]:
+    return {
+        "type": "way",
+        "id": identifier,
+        "tags": {"natural": "water"},
+        "geometry": [{"lon": x, "lat": y} for x, y in points],
+    }
+
+
+def test_water_polygon_and_relation_preserve_islands() -> None:
+    outer = [(0, 0), (4, 0), (4, 4), (0, 4), (0, 0)]
+    inner = [(1, 1), (3, 1), (3, 3), (1, 3), (1, 1)]
+    members = [
+        {"type": "way", "ref": 1, "role": "outer", "geometry": water_way(outer)["geometry"]},
+        {"type": "way", "ref": 2, "role": "inner", "geometry": water_way(inner)["geometry"]},
+    ]
+    relation = {
+        "type": "relation",
+        "tags": {"natural": "water", "type": "multipolygon"},
+        "members": members,
+    }
+    geometry = water_areas({"elements": [water_way(outer), relation]})
+    assert geometry.area == 12
+    assert not geometry.covers(box(1.5, 1.5, 2.5, 2.5))
+    assert water_areas({"elements": [water_way(outer)]}).area == 16
+    # Relations often consist of several open ways joined into an outer ring.
+    members[0]["geometry"] = water_way(outer[:3])["geometry"]
+    members.append(
+        {"type": "way", "ref": 3, "role": "outer", "geometry": water_way(outer[2:])["geometry"]}
+    )
+    assert water_areas({"elements": [relation]}).area == 12
+    assert water_areas({"elements": []}).is_empty
+    with pytest.raises(ValueError, match="unclosed"):
+        water_areas({"elements": [water_way(outer[:3])]})
+    members.pop()
+    with pytest.raises(ValueError, match="incomplete"):
+        water_areas({"elements": [relation]})
+
+
+def test_land_mean_excludes_water_but_reports_missing_land(
+    tmp_path: Path,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    path = tmp_path / "terrain.tif"
+    bounds = Bounds(0, 0, 4, 4)
+    values = np.full((4, 4), 100, dtype="float32")
+    values[:, :2] = -50  # These are valid water elevations, which must not enter the mean.
+    values[0, 2] = -9999  # Missing land is a separate coverage issue.
+    with rasterio.open(
+        path,
+        "w",
+        driver="GTiff",
+        width=4,
+        height=4,
+        count=1,
+        dtype="float32",
+        crs="EPSG:4326",
+        transform=from_bounds(*bounds.values, 4, 4),
+        nodata=-9999,
+    ) as dataset:
+        dataset.write(values, 1)
+    with Elevation(path, "nap") as elevation:
+        assert elevation.mean_ground(bounds, 4, water=box(0, 0, 2, 4)) == 100
+        assert elevation.water_samples == 8
+        assert elevation.land_samples == 8
+        assert elevation.valid_land_samples == 7
+        assert elevation.land_coverage == 7 / 8
+        assert "87.5%" in caplog.text
+        assert "Missing land is not treated as water" in caplog.text
+        assert elevation.mean_ground(bounds, 4, water=box(0, 0, 4, 4)) is None
+
+
+def test_plot_auto_ahn_and_warning_fallback(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    db = tmp_path / "flights.sqlite"
+    with database(db) as connection:
+        initialize(connection)
+        session = create_session(connection, BOUNDS, 2000, 30, 100)
+        save_observations(
+            connection,
+            session,
+            [
+                Observation("abc123", "TEST", 110, 4.79, 52.24, 900, 1043),
+                Observation("abc123", "TEST", 140, 4.85, 52.24, 900, 1043),
+            ],
+            150,
+        )
+        checkpoint(connection, session.id, 150, "stopped")
+    local = tmp_path / "map.json"
+    local.write_text(
+        json.dumps(
+            {
+                "elements": [
+                    {
+                        "type": "way",
+                        "geometry": [{"lon": 4.79, "lat": 52.24}, {"lon": 4.85, "lat": 52.24}],
+                    },
+                ]
+            }
+        )
+    )
+    monkeypatch.setattr("flight_map.ahn.request_bytes", Mock(return_value=ahn_bytes(-3)))
+    output = tmp_path / "map.svg"
+    assert render(db, session.id, output, local_map=local, cache=tmp_path / "cache") == 1
+    text = output.read_text()
+    assert "PDOK AHN DTM" in text and "Terrain reference: nap" in text
+    root = ET.parse(output).getroot()
+    flight = root.find(f".//{{{SVG_NS}}}g[@id='flight-1']")
+    assert flight is not None
+    assert flight.attrib["data-average-barometric-altitude-metres"] == "900.000"
+    assert "altitude(s) unavailable" not in caplog.text
+    assert "ground elevation in metres: -3.0" in text
+    monkeypatch.setattr("flight_map.ahn.request_bytes", Mock(return_value=ahn_bytes(100)))
+    changed_output = tmp_path / "changed-ground.svg"
+    assert (
+        render(
+            db,
+            session.id,
+            changed_output,
+            local_map=local,
+            cache=tmp_path / "cache",
+            refresh_terrain=True,
+        )
+        == 1
+    )
+    changed_flight = ET.parse(changed_output).getroot().find(f".//{{{SVG_NS}}}g[@id='flight-1']")
+    assert changed_flight is not None
+    assert changed_flight.attrib["data-average-barometric-altitude-metres"] == "900.000"
+    assert "ground elevation in metres: 100.0" in changed_output.read_text()
+    monkeypatch.setattr("flight_map.ahn.request_bytes", Mock(side_effect=RuntimeError("offline")))
+    assert (
+        render(
+            db,
+            session.id,
+            tmp_path / "unavailable.svg",
+            local_map=local,
+            cache=tmp_path / "other-cache",
+        )
+        == 1
+    )
+    assert "AHN height map unavailable" in caplog.text
+    assert "Average land elevation unavailable" in caplog.text
+    unavailable = ET.parse(tmp_path / "unavailable.svg").getroot()
+    unchanged = unavailable.find(f".//{{{SVG_NS}}}g[@id='flight-1']")
+    assert unchanged is not None
+    assert unchanged.attrib["data-average-barometric-altitude-metres"] == "900.000"
+    invalid_water = json.loads(local.read_text())
+    invalid_water["elements"].append(water_way([(4.80, 52.24), (4.81, 52.25)]))
+    local.write_text(json.dumps(invalid_water))
+    assert render(db, session.id, tmp_path / "bad-water.svg", local_map=local, no_terrain=True) == 1
+    assert "unclosed outline" in caplog.text
+    assert "Reported flight altitudes are unaffected" in caplog.text
+    downloader = Mock(side_effect=AssertionError("must stay offline"))
+    monkeypatch.setattr("flight_map.plot.terrain_data", downloader)
+    assert render(db, session.id, tmp_path / "disabled.svg", local_map=local, no_terrain=True) == 1
+    downloader.assert_not_called()
+    with database(db) as connection:
+        with connection:
+            connection.execute("UPDATE observations SET baro_altitude = NULL")
+    missing_output = tmp_path / "missing-baro.svg"
+    assert render(db, session.id, missing_output, local_map=local, no_terrain=True) == 1
+    missing_flight = ET.parse(missing_output).getroot().find(f".//{{{SVG_NS}}}g[@id='flight-1']")
+    assert missing_flight is not None
+    assert missing_flight.attrib["data-average-barometric-altitude-metres"] == "unavailable"
+    assert "reported barometric altitude is missing" in caplog.text
+    assert "no geometric-altitude fallback" in caplog.text
+    with pytest.raises(ValueError, match="cannot be combined"):
+        render(db, session.id, tmp_path / "bad.svg", no_terrain=True, refresh_terrain=True)
+
+
+def test_binary_download_errors_and_size_limit(monkeypatch: pytest.MonkeyPatch) -> None:
+    from urllib.error import URLError
+
+    response = Mock()
+    response.__enter__ = Mock(return_value=response)
+    response.__exit__ = Mock(return_value=False)
+    response.read.return_value = b"12345"
+    fetch = Mock(return_value=response)
+    monkeypatch.setattr("flight_map.network.urlopen", fetch)
+    assert request_bytes("https://example.invalid") == b"12345"
+    with pytest.raises(RuntimeError, match="limit"):
+        request_bytes("https://example.invalid", max_bytes=4)
+    fetch.side_effect = URLError("offline")
+    with pytest.raises(RuntimeError, match="download failed"):
+        request_bytes("https://example.invalid")

tests/test_flight_map.py

@@ -105,8 +105,8 @@ def local_map(tmp_path: Path) -> Path:
 
 
 def test_default_and_reversed_bounds() -> None:
-    assert DEFAULT_BOUNDS.values == (4.784498, 52.232025, 4.863525, 52.253985)
-    assert Bounds.corners(4.863525, 52.253985, 4.784498, 52.232025) == DEFAULT_BOUNDS
+    assert DEFAULT_BOUNDS.values == (4.784, 52.232, 4.864, 52.258)
+    assert Bounds.corners(4.864, 52.258, 4.784, 52.232) == DEFAULT_BOUNDS
     assert DEFAULT_BOUNDS.contains(4.83, 52.24)
     expanded = DEFAULT_BOUNDS.expand(2000)
     assert expanded.west < DEFAULT_BOUNDS.west and expanded.north > DEFAULT_BOUNDS.north
@@ -174,29 +174,22 @@ def test_database_readonly(recorded: tuple[Path, Session]) -> None:
             connection.execute("DELETE FROM observations")
 
 
-def test_terrain_and_datum_conversion(tmp_path: Path) -> None:
+def test_local_terrain_summary(tmp_path: Path) -> None:
     terrain = raster(tmp_path / "terrain.tif", -2)
-    geoid = raster(tmp_path / "geoid.tif", 43)
-    with Elevation(terrain, "orthometric", geoid, True) as elevation:
+    with Elevation(terrain, "orthometric") as elevation:
         assert elevation.mean_ground(DEFAULT_BOUNDS, 4) == -2
-        assert elevation.aircraft_altitude(observation()) == 1000
-    with Elevation(terrain, "orthometric", geoid, False) as elevation:
-        assert elevation.aircraft_altitude(observation()) is None
-    with Elevation(terrain, "orthometric", None, True) as elevation:
-        assert elevation.aircraft_altitude(observation()) is None
-    with Elevation(terrain, "wgs84-ellipsoid", None, True) as elevation:
-        assert elevation.aircraft_altitude(observation()) == 1043
-        assert elevation.aircraft_altitude(observation(geo_altitude=None)) is None
 
 
-def test_terrain_missing_data(tmp_path: Path) -> None:
+def test_terrain_missing_data(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None:
     terrain = raster(tmp_path / "terrain.tif", -9999)
     with Elevation(terrain, "wgs84-ellipsoid") as elevation:
         assert elevation.mean_ground(DEFAULT_BOUNDS, 4) is None
     small = Bounds(4.80, 52.24, 4.81, 52.25)
     raster(terrain, 0, bounds=small)
     with Elevation(terrain, "wgs84-ellipsoid") as elevation:
-        assert elevation.mean_ground(DEFAULT_BOUNDS, 4) is None
+        assert elevation.mean_ground(DEFAULT_BOUNDS, 16) == 0
+        assert 0 < elevation.land_coverage < 1
+    assert "averaging valid land only" in caplog.text
 
 
 def test_equal_area_ground_average(tmp_path: Path) -> None:
@@ -209,18 +202,17 @@ def test_equal_area_ground_average(tmp_path: Path) -> None:
         assert elevation.mean_ground(DEFAULT_BOUNDS, 10) == pytest.approx(5)
 
 
-def test_height_is_time_weighted_and_excludes_extensions() -> None:
+def test_reported_altitude_is_time_weighted_and_excludes_extensions() -> None:
     bounds = Bounds(0, 0, 2, 1)
     observations = [
-        observation(longitude=-1, latitude=0.5, timestamp=T0, geo_altitude=0),
-        observation(longitude=1, latitude=0.5, timestamp=T0 + 20, geo_altitude=200),
-        observation(longitude=3, latitude=0.5, timestamp=T0 + 60, geo_altitude=600),
+        observation(longitude=-1, latitude=0.5, timestamp=T0, baro_altitude=0),
+        observation(longitude=1, latitude=0.5, timestamp=T0 + 20, baro_altitude=200),
+        observation(longitude=3, latitude=0.5, timestamp=T0 + 60, baro_altitude=600),
     ]
-    with Elevation(None, "wgs84-ellipsoid", None, True) as elevation:
-        tracks = make_tracks(observations, Projection(bounds), elevation, 10, 120)
+    tracks = make_tracks(observations, Projection(bounds), 120)
     # Inside: 10s at mean altitude 150 + 20s at mean altitude 300.
     assert len(tracks) == 1
-    assert tracks[0].average_height == pytest.approx(240)
+    assert tracks[0].average_altitude == pytest.approx(250)
     assert tracks[0].start == pytest.approx(T0 + 10)
     assert tracks[0].end == pytest.approx(T0 + 40)
 
@@ -229,13 +221,31 @@ def test_stationary_positions_and_missing_altitude() -> None:
     items = [
         observation(),
         observation(timestamp=T0 + 25),
+        observation(timestamp=T0 + 40, longitude=4.82, baro_altitude=None),
+    ]
+    tracks = make_tracks(items, Projection(DEFAULT_BOUNDS), 120)
+    assert len(tracks) == 1 and tracks[0].average_altitude is None
+    assert make_tracks(items[:2], Projection(DEFAULT_BOUNDS), 120) == []
+
+
+def test_reported_altitude_does_not_require_geometric_altitude() -> None:
+    items = [
+        observation(geo_altitude=None),
         observation(timestamp=T0 + 40, longitude=4.82, geo_altitude=None),
     ]
-    with Elevation(None, "wgs84-ellipsoid", None, True) as elevation:
-        tracks = make_tracks(items, Projection(DEFAULT_BOUNDS), elevation, -2, 120)
-    assert len(tracks) == 1 and tracks[0].average_height is None
-    with Elevation(None) as elevation:
-        assert make_tracks(items[:2], Projection(DEFAULT_BOUNDS), elevation, None, 120) == []
+    tracks = make_tracks(items, Projection(DEFAULT_BOUNDS), 120)
+    assert len(tracks) == 1
+    assert tracks[0].average_altitude == 900
+
+
+@pytest.mark.parametrize("altitude", [0, -50, 1234])
+def test_zero_negative_and_positive_reported_altitudes_are_preserved(altitude: float) -> None:
+    items = [
+        observation(baro_altitude=altitude, geo_altitude=9999),
+        observation(timestamp=T0 + 40, longitude=4.82, baro_altitude=altitude, geo_altitude=9999),
+    ]
+    tracks = make_tracks(items, Projection(DEFAULT_BOUNDS), 120)
+    assert tracks[0].average_altitude == pytest.approx(altitude)
 
 
 def test_track_splitting_and_ground_filter() -> None:
@@ -275,7 +285,6 @@ def test_end_to_end_offline_repeatable(
                 local_map=local_map,
                 terrain=terrain,
                 terrain_reference="wgs84-ellipsoid",
-                confirm_geometric_ellipsoid=True,
             )
             == 1
         )
@@ -296,12 +305,13 @@ def test_end_to_end_offline_repeatable(
     assert flight_paths and all("C " in element.attrib["d"] for element in flight_paths)
     flight = root.find(f".//{{{SVG_NS}}}g[@id='flight-1']")
     assert flight is not None and flight.attrib["data-callsign"] == "KLM123"
-    assert flight.attrib["data-average-height-metres"] != "unavailable"
+    assert flight.attrib["data-average-barometric-altitude-metres"] == "900.000"
+    assert flight.attrib["data-altitude-source"] == "barometric"
     for element in root.iter(f"{{{SVG_NS}}}path"):
         assert "clip-path" not in element.attrib
         for x, y in re.findall(r"(-?\d+\.\d+),(-?\d+\.\d+)", element.attrib["d"]):
             assert 0 <= float(x) <= 297 and 0 <= float(y) <= 210
-    assert "HEIGHT" not in output.read_text()  # Lettering is geometry, not SVG text.
+    assert "BARO ALT" not in output.read_text()  # Lettering is geometry, not SVG text.
     with pytest.raises(ValueError, match="already exists"):
         render(path, session.id, output, local_map=local_map)
 
@@ -448,7 +458,7 @@ def test_opensky_parsing_and_auth(monkeypatch: pytest.MonkeyPatch) -> None:
     client = OpenSky()
     items = client.observations(DEFAULT_BOUNDS)
     assert items[0].callsign == "KLM123" and items[0].geo_altitude == 1043
-    assert "lamin=52.232025" in fetch.call_args.args[0]
+    assert f"lamin={DEFAULT_BOUNDS.south}" in fetch.call_args.args[0]
     state[5] = None
     assert client.observations(DEFAULT_BOUNDS) == []
     fetch.return_value = {"states": None}
@@ -521,6 +531,7 @@ def test_cli_validation_and_plot(
                 session.id,
                 "--map-data",
                 str(local_map),
+                "--no-terrain",
                 "--output",
                 str(tmp_path / "cli.svg"),
             ]