From b692f941224d3e42fffc70a4fbc539c8657d1f5d Mon Sep 17 00:00:00 2001 From: Olaf Date: Wed, 21 Oct 2026 09:24:12 +0200 Subject: [PATCH] height correction, new defalt --- README.md | 132 ++++++++++------- REQUIREMENTS.md | 69 +++++---- examples/make_demo.py | 5 +- flight_map/ahn.py | 108 ++++++++++++++ flight_map/cli.py | 28 ++-- flight_map/maps.py | 96 +++++++++--- flight_map/network.py | 12 ++ flight_map/plot.py | 84 ++++++++--- flight_map/terrain.py | 70 +++++---- flight_map/tracks.py | 15 +- tests/test_ahn.py | 305 +++++++++++++++++++++++++++++++++++++++ tests/test_flight_map.py | 73 ++++++---- 12 files changed, 792 insertions(+), 205 deletions(-) create mode 100644 flight_map/ahn.py create mode 100644 tests/test_ahn.py diff --git a/README.md b/README.md index 5695cdd..fd2577b 100644 --- a/README.md +++ b/README.md @@ -23,8 +23,8 @@ The default map rectangle is in Uithoorn, Netherlands: | Corner | Latitude | Longitude | | --- | --- | --- | -| Southwest | 52.232025 | 4.784498 | -| Northeast | 52.253985 | 4.863525 | +| Southwest | 52.232 | 4.784 | +| Northeast | 52.258 | 4.864 | OpenSky supports OAuth client credentials. Obtain your own credentials through OpenSky and set them in your shell, **not in source code**: @@ -116,7 +116,7 @@ remain straight; duplicate positions are handled without invalid coordinates. Smoothing never joins separate tracks or fills long recording gaps. Curves are interpolations, not evidence of the aircraft's precise motion between samples. The actual curves are clipped geometrically to the output rectangle, and their -in-map portions determine the time-weighted height calculation. +in-map portions determine the time-weighted reported-altitude calculation. ### Map cache and offline plotting @@ -126,8 +126,9 @@ unfilled lines. Minor pedestrian/cycle paths, steps, proposed roads, and roads under construction are omitted from the drawing. Building outlines whose shorter dimension would be smaller than three pen widths are omitted to prevent dense housing from turning into ink blobs; larger paper or a finer pen retains more -buildings without another download. Multipolygon relations are not assembled in this initial version; -only returned way geometry is drawn. +buildings without another download. Water multipolygon relations (including +islands) are assembled for the land-elevation mask; the map drawing itself uses +returned way geometry. Downloaded map data is cached in `.flight-map-cache/`. The cache key includes the service URL, exact map bounds, and feature query. Different time ranges, @@ -151,59 +152,84 @@ For entirely offline plotting, pass a local Overpass JSON export containing ```sh flight-map plot --db flights.sqlite --session SESSION_ID --output offline.svg \ - --map-data local-map.json + --map-data local-map.json --no-terrain ``` -### Height above the map's average ground level +Omit `--no-terrain` when using cached automatic AHN terrain or supply a local +`--terrain` raster for the separate land-elevation summary offline. For a complete water mask, +local exports must include closed water ways and water multipolygon relations +with full member geometry, including areas enclosing the requested map. -**Without suitable elevation inputs the map still plots, but heights are `N/A` -and the program emits warnings.** It never substitutes pressure altitude, zero -ground elevation, or sea-level altitude. +### Reported aircraft altitude -The calculation is: +Flight labels use **average reported barometric altitude (pressure altitude)**, +in metres, from OpenSky's `baro_altitude` field. It is time-weighted over the +smoothed flight path's portion inside the saved map bounds, using linear +interpolation of altitude between observation timestamps. Outside-map extensions +do not contribute. -```text -height = time-weighted mean aircraft altitude inside the saved map bounds - minus area-weighted mean ground elevation inside those same bounds -``` +No terrain elevation is subtracted and no geoid, pressure, or vertical-reference +correction is applied. This is not height above ground, terrain clearance, or +necessarily true altitude above sea level. Geometric altitude is still stored +but is never substituted for missing barometric data. If a contributing segment +has missing barometric altitude, the track's altitude is `N/A` with a warning. +Zero and negative reported altitudes remain valid. -Outside-map extensions do not contribute. Heights are in metres, not terrain -clearance under the aircraft. +Flight altitude does not depend on terrain downloads or the water mask. +The former `--geoid` and `--confirm-geometric-ellipsoid` options have been removed. +SVG metadata uses `data-average-barometric-altitude-metres` and +`data-altitude-source="barometric"` instead of `data-average-height-metres`. -Supply a terrain GeoTIFF whose first band contains ground elevation in metres, -has a CRS, and covers the entire map rectangle. Its declared vertical reference -must be known. Sampling uses an equal-area 64 by 64 geographic grid and honors -nodata, raster band scale, and offset. Any missing sampled ground value makes -the mean unavailable. Choose terrain resolution appropriate to the map; the -grid is an approximation, not a survey-grade terrain integral. +### Separate average land-elevation summary -For terrain already referenced to the WGS84 ellipsoid: +For Dutch map bounds, plotting automatically downloads the **AHN digital terrain +model (DTM)** through the [PDOK WCS](https://service.pdok.nl/rws/ahn/wcs/v1_0?service=WCS&request=GetCapabilities). +It uses coverage `dtm_05m`, not a surface model containing roofs and vegetation. +The service currently describes this coverage as AHN4. Data is requested in +RD New (EPSG:28992) as a 512 by 512 nearest-neighbour GeoTIFF subset of the +source's 0.5 m raster. The downloaded subset is a sampled overview, not the +full-resolution source. Elevations are metres relative to **NAP**. ```sh -flight-map plot --db flights.sqlite --session SESSION_ID --output heights.svg \ - --terrain terrain-ellipsoid.tif --terrain-reference wgs84-ellipsoid \ - --confirm-geometric-ellipsoid +flight-map plot --db flights.sqlite --session SESSION_ID --output map.svg +flight-map plot --db flights.sqlite --session SESSION_ID --output refreshed.svg \ + --refresh-terrain ``` -For orthometric terrain, also supply a geoid separation GeoTIFF in metres, -representing **N = h - H**, matching the terrain's vertical datum and covering -the track observations used for conversion: +Terrain is cached under `--cache/terrain/` (by default +`.flight-map-cache/terrain/`), keyed by the PDOK endpoint, coverage, projected +bounds, resolution, and request parameters. Validated downloads atomically replace +cache entries; refresh failures preserve the previous file but warn and leave +land elevation unavailable for that run. Files contain provenance and NAP metadata. +There is no automatic expiry. A changed vector-map query also gets a new cache +key; the first run after the water-mask update downloads fresh OSM data. -```sh -flight-map plot --db flights.sqlite --session SESSION_ID --output heights.svg \ - --terrain terrain.tif --terrain-reference orthometric \ - --geoid matching-geoid.tif --confirm-geometric-ellipsoid -``` +Outside AHN's coverage, on service/download failures, or when no valid land +elevations are available, the program **warns and continues plotting**. Ground +elevation is then `N/A`; reported aircraft altitudes remain available independently. +Use `--no-terrain` to disable the separate land-elevation summary and its downloads. +The land average is informational only and never corrects aircraft altitude. -The application computes `H = h - N` for each geometric aircraft altitude. -OpenSky documents its field as geometric altitude but does not establish every -observation's datum or accuracy. The confirmation flag is a deliberate assertion -by the operator that treating these heights as WGS84 ellipsoidal is appropriate. -Do not set it without verifying your source's interpretation. A missing geometric -altitude or geoid value on a contributing segment makes that track's height -unavailable. Pressure altitude is recorded for provenance, not used for this -calculation. For Dutch terrain in NAP, use a conversion consistent with NAP; -do not assume an arbitrary global geoid matches it. +Sampling uses an equal-area 64 by 64 geographic grid. Mapped OSM water areas +(`natural=water` or `waterway=riverbank`) are excluded, including multipolygon +water areas with islands retained as land. Unmapped water and coastline-derived +sea areas are not inferred. Water masking is only as complete as the OSM data. +If a water outline is unclosed or a multipolygon is incomplete, the program warns +and leaves land elevation unavailable rather than silently +counting that water as land. Map and flight geometry still render. + +Valid samples on the remaining land are averaged, honoring raster nodata, scale, +and offset. Missing land samples are **not** assumed to be water: they are +excluded with a warning showing the valid land coverage percentage and sample +counts. Even small coverage gaps produce a warning. Coverage is included in +the plotted legend and SVG metadata; low coverage can bias the average. No +arbitrary minimum is imposed: no valid land samples means `N/A`. The grid is +an approximation, not a survey-grade terrain integral. + +You can override automatic AHN with a local terrain GeoTIFF whose first band +contains ground elevation in metres and has a CRS. Declare its vertical reference +with `--terrain-reference` (`nap`, `orthometric`, or `wgs84-ellipsoid`). +`--refresh-terrain` only applies to automatic AHN, not local input. ### Pen-plotter settings @@ -224,7 +250,7 @@ with an original single-stroke alphabet: no installed fonts, raster images, filled glyphs, or SVG clipping support are required. Each track has a numeric identifier and a legend entry with UTC dates/times, -callsign, ICAO identifier, and average height. For dense recordings, select +callsign, ICAO identifier, and average reported barometric altitude. For dense recordings, select shorter periods or larger paper. The program fails explicitly if the legend does not fit or a track identifier cannot be placed legibly. Flight identifiers avoid each other; intersections with map geometry or flight paths are not fully @@ -237,15 +263,15 @@ rejected. ## Offline synthetic demo -This creates **invented** tracks, map features, terrain, and geoid data, for +This creates **invented** tracks, map features, and terrain data, for checking the complete workflow without network access: ```sh python examples/make_demo.py demo ``` -Run the plotting command printed by the script. Its heights use synthetic, -known-compatible references; these inputs are not a real Uithoorn dataset. +Run the plotting command printed by the script. Altitude labels use the synthetic +barometric observations directly; these inputs are not a real Uithoorn dataset. ## Development @@ -258,8 +284,8 @@ mypy flight_map ``` Tests cover recording and resume settings, persistence, deduplication, bounds, -projection, track gaps, clipping, time-weighted heights, missing terrain, vertical -references, cache reuse/refresh/failure, and repeatable offline SVG generation +projection, track gaps, clipping, time-weighted reported barometric altitude, +missing altitude and terrain, cache reuse/refresh/failure, and repeatable offline SVG generation without mutating the database. ## Data policies and attribution @@ -276,8 +302,10 @@ without mutating the database. Cite Matthias Schaefer, Martin Strohmeier, Vincent Lenders, Ivan Martinovic, and Matthias Wilhelm, "Bringing Up OpenSky: A Large-scale ADS-B Sensor Network for Research", IPSN 2014, pp. 83-94, when publishing derived work. -- Terrain and geoid: supplied by the operator; follow their separate licenses - and attribution requirements when distributing outputs. +- Local terrain: supplied by the operator; follow its separate license + and attribution requirements when distributing outputs. Automatic AHN DTM + through PDOK is CC0; its source and NAP reference are included in SVG metadata + and the plotted attribution. Map and flight attribution is included as plotter paths, with further provenance in SVG metadata. This is an artistic recording/plotting tool, not a navigation diff --git a/REQUIREMENTS.md b/REQUIREMENTS.md index 3d5ddbe..1a02e62 100644 --- a/REQUIREMENTS.md +++ b/REQUIREMENTS.md @@ -45,9 +45,9 @@ a physical pen plotter. of the map, validate coordinates, and normalize their order. - Provide a predefined default rectangular map area that includes the city of Uithoorn in the Netherlands. Explicitly supplied corners override this default. -- Default southwest corner: latitude 52.232025, longitude 4.784498. - Default northeast corner: latitude 52.253985, longitude 4.863525. - Command-line bounds use longitude first: 4.784498 52.232025 4.863525 52.253985. +- Default southwest corner: latitude 52.232, longitude 4.784. + Default northeast corner: latitude 52.258, longitude 4.864. + Command-line bounds use longitude first: 4.784 52.232 4.864 52.258. - Store the actual bounds used, including when the default is selected, in the recording session. Later plotting must use those stored bounds rather than a potentially changed application default. @@ -114,31 +114,41 @@ a physical pen plotter. - Do not smooth across track gaps, on-ground interruptions, or flight changes. Handle repeated positions without generating invalid curve coordinates. - Clip the actual curves to the output bounds, not just their sample points, - and use the same curved in-map portions for time and average-height selection. + and use the same curved in-map portions for time and average-altitude selection. - Extend the visible tracks a little beyond the map boundary using actual recorded positions from the surrounding area, not fabricated extrapolation. - Keep the map background within its requested bounds and provide space in the SVG for the extended tracks and their labels. - Identify each flight track with its date, time, callsign or flight number when - available, and average height above the map's average ground elevation. + available, and average reported barometric (pressure) altitude in metres. - Do not present an OpenSky callsign as a verified passenger-facing flight number. Callsign-to-flight-number lookup is not required for the initial version. - Calculate the map's average ground elevation from terrain elevation data - covering the requested map bounds, excluding the surrounding collection area. + for landmass within the requested map bounds, excluding mapped water and the + surrounding collection area. Preserve islands in water multipolygon relations. Use an area-weighted mean or an equivalent equal-area sampling method. -- Calculate each track's average height as its average aircraft altitude over - the portion inside the requested map bounds minus the map's average ground - elevation. Outside-map extensions must not affect this calculation. -- Use compatible units and a common vertical reference for aircraft altitude - and terrain elevation, applying conversions where needed. Do not subtract - terrain elevation directly from uncorrected barometric pressure altitude. -- Label the result explicitly as average height above the map's average ground - level, with units. This is not height above the terrain directly beneath each - aircraft position and must not be presented as terrain clearance. -- Indicate when the height cannot be calculated because compatible aircraft - altitude or sufficient terrain data is unavailable; do not silently substitute - altitude above sea level or assume zero ground elevation. +- Average valid land samples only. Report the percentage and counts of sampled + land with valid elevations, warn about missing land samples, and never classify + missing terrain as water. Show land coverage in the SVG legend and metadata. +- Obtain AHN DTM height maps automatically through PDOK during plotting for the + Netherlands. Cache validated GeoTIFF subsets persistently and atomically, + record source and NAP reference, and support explicit refresh and disabling + of automatic terrain retrieval. Explicit local terrain inputs take precedence. +- If AHN height maps are unavailable or the area is outside coverage, warn + explicitly and continue rendering with the land-elevation summary unavailable. + Reported aircraft altitude must remain independent of terrain availability. +- Calculate each track's time-weighted average reported barometric altitude + over the portion inside the requested map bounds. Outside-map extensions + must not affect this calculation. +- Do not subtract ground elevation or apply geoid, pressure, or vertical-reference + corrections to aircraft altitude. The average land elevation is a separate, + informational summary only. +- Label aircraft values explicitly as reported barometric altitude with units, + not height above ground, terrain clearance, or verified altitude above sea level. +- Missing barometric altitude on a contributing segment makes that track's + average unavailable, with an explicit warning. Do not fall back to geometric + altitude. Zero and negative reported values are valid. - Make labels readable and indicate unavailable metadata explicitly. ## Implementation and validation @@ -148,8 +158,8 @@ a physical pen plotter. session selection, map generation, configuration, data-source limitations, and attribution. - Test coordinate handling, projection, local recording, track selection, - average ground elevation, vertical-reference compatibility, average-height - calculation, missing elevation data, and SVG generation. + average land elevation, average reported barometric altitude, missing altitude + and elevation data, terrain-independent flight labels, and SVG generation. - Test persistence of session bounds and observation associations, plotting after collection has stopped, and repeated plotting without changing the recorded flight data. Verify that collection does not require map or terrain @@ -172,15 +182,16 @@ a physical pen plotter. - Default to A4 landscape (297 by 210 mm), 12 mm page margins, a 0.3 mm pen, and track extensions of up to 4 mm beyond the map border. - Collect every 30 seconds within a 2,000 m margin around the saved map area. -- Display timestamps in UTC using date and 24-hour time, and heights in metres. -- Accept local terrain GeoTIFFs with a declared vertical reference. For - orthometric terrain, accept a matching geoid-separation raster for converting - ellipsoidal aircraft heights. Require explicit confirmation of the geometric - altitude interpretation before calculating heights; otherwise label them - unavailable. Never silently substitute barometric altitude. -- Estimate average ground elevation using a 64 by 64 equal-area sampling grid. - Any missing terrain sample makes the average unavailable. -- Use time-weighted aircraft altitude with linear interpolation between recorded +- Display timestamps in UTC using date and 24-hour time, and altitudes in metres. +- Use PDOK AHN DTM `dtm_05m` via WCS for Dutch areas, requesting a 512 by 512 + nearest-neighbour subset in EPSG:28992, with elevations in metres relative + to NAP. Accept local terrain GeoTIFFs with a declared vertical reference for + the separate land-elevation summary. No geoid or aircraft datum conversion + inputs are required or used. +- Estimate average land elevation using a 64 by 64 equal-area sampling grid. + Exclude mapped water, average available land elevations, and warn about missing + land coverage. No valid land samples makes the average unavailable. +- Use time-weighted reported barometric altitude with linear interpolation between recorded timestamps, restricted to the smoothed path's portion inside the map. Use centripetal Catmull-Rom interpolation converted to cubic Bezier curves for flight paths. Split tracks at changes diff --git a/examples/make_demo.py b/examples/make_demo.py index d9b3b3a..39e512b 100644 --- a/examples/make_demo.py +++ b/examples/make_demo.py @@ -78,7 +78,7 @@ def main() -> None: } ) ) - for name, value in (("terrain.tif", -2), ("geoid.tif", 43)): + for name, value in (("terrain.tif", -2),): with rasterio.open( folder / name, "w", @@ -95,8 +95,7 @@ def main() -> None: print( f"flight-map plot --db '{db}' --session {session.id} " f"--map-data '{folder / 'map.json'}' --terrain '{folder / 'terrain.tif'}' " - f"--terrain-reference orthometric --geoid '{folder / 'geoid.tif'}' " - f"--confirm-geometric-ellipsoid --output '{folder / 'demo.svg'}'" + f"--terrain-reference orthometric --output '{folder / 'demo.svg'}'" ) diff --git a/flight_map/ahn.py b/flight_map/ahn.py new file mode 100644 index 0000000..4f51d29 --- /dev/null +++ b/flight_map/ahn.py @@ -0,0 +1,108 @@ +"""PDOK AHN DTM coverage in RD New, elevations in metres relative to NAP.""" + +import hashlib +import logging +import math +import os +import tempfile +from pathlib import Path +from urllib.parse import urlencode + +import rasterio +from pyproj import Transformer +from rasterio.errors import RasterioError + +from flight_map.geo import Bounds +from flight_map.network import request_bytes + +LOGGER = logging.getLogger(__name__) +WCS_URL = "https://service.pdok.nl/rws/ahn/wcs/v1_0" +COVERAGE = "dtm_05m" +AHN_EXTENT = Bounds(3.20030320333955, 50.728765211722, 7.273444085294, 53.555958411015) +SIZE = 512 + + +def coverage_url(bounds: Bounds) -> str: + transform = Transformer.from_crs("EPSG:4326", "EPSG:28992", always_xy=True) + west, south, east, north = transform.transform_bounds(*bounds.values, densify_pts=21) + bbox = ",".join( + str(v) for v in (math.floor(west), math.floor(south), math.ceil(east), math.ceil(north)) + ) + return ( + WCS_URL + + "?" + + urlencode( + { + "service": "WCS", + "version": "1.0.0", + "request": "GetCoverage", + "coverage": COVERAGE, + "crs": "EPSG:28992", + "response_crs": "EPSG:28992", + "bbox": bbox, + "width": SIZE, + "height": SIZE, + "format": "GEOTIFF", + "interpolation": "nearest neighbor", + } + ) + ) + + +def validate_raster(path: Path) -> None: + with rasterio.open(path) as dataset: + if dataset.driver != "GTiff" or dataset.crs != rasterio.crs.CRS.from_epsg(28992): + raise ValueError("AHN response is not an RD New GeoTIFF.") + if dataset.count != 1 or dataset.width != SIZE or dataset.height != SIZE: + raise ValueError("AHN response has unexpected raster dimensions.") + dataset.read(1, masked=True) + + +def terrain_data(bounds: Bounds, cache: Path, *, refresh: bool = False) -> Path | None: + if not ( + AHN_EXTENT.contains(bounds.west, bounds.south) + and AHN_EXTENT.contains(bounds.east, bounds.north) + ): + LOGGER.warning("AHN height map unavailable: map bounds extend outside Dutch AHN coverage.") + return None + url = coverage_url(bounds) + key = hashlib.sha256(url.encode()).hexdigest() + path = cache / f"ahn-{key}.tif" + temporary: Path | None = None + try: + if path.exists() and not refresh: + validate_raster(path) + LOGGER.info("Reading cached AHN DTM (NAP): %s", path) + return path + LOGGER.info("Downloading AHN DTM height map from PDOK for %s", bounds.values) + content = request_bytes(url) + cache.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=cache, suffix=".tif", delete=False) as stream: + temporary = Path(stream.name) + stream.write(content) + stream.flush() + os.fsync(stream.fileno()) + validate_raster(temporary) + with rasterio.open(temporary, "r+") as dataset: + dataset.update_tags( + source=WCS_URL, + coverage=COVERAGE, + vertical_reference="NAP", + units="metres", + license="CC0", + request_url=url, + ) + os.replace(temporary, path) + LOGGER.info("Cached AHN DTM (NAP): %s", path) + return path + except (RuntimeError, OSError, ValueError, RasterioError) as exc: + LOGGER.warning( + "AHN height map unavailable: %s. Land elevation will be N/A; " + "reported flight altitudes are unaffected. " + "Use --refresh-terrain to retry a damaged cache entry.", + exc, + ) + return None + finally: + if temporary and temporary.exists(): + temporary.unlink() diff --git a/flight_map/cli.py b/flight_map/cli.py index 95e8a76..15e439e 100644 --- a/flight_map/cli.py +++ b/flight_map/cli.py @@ -96,19 +96,19 @@ def parser() -> argparse.ArgumentParser: "--terrain", type=Path, help="Local terrain GeoTIFF; elevations must be in metres" ) plot.add_argument( - "--terrain-reference", - choices=("unknown", "wgs84-ellipsoid", "orthometric"), - default="unknown", - ) - plot.add_argument( - "--geoid", - type=Path, - help="Metre-valued h-H raster matching the terrain's orthometric datum", - ) - plot.add_argument( - "--confirm-geometric-ellipsoid", + "--no-terrain", action="store_true", - help="Confirm OpenSky geometric heights can be treated as WGS84 ellipsoidal", + help="Disable the separate land-elevation summary and automatic AHN download", + ) + plot.add_argument( + "--refresh-terrain", + action="store_true", + help="Refresh the cached PDOK AHN DTM for this map", + ) + plot.add_argument( + "--terrain-reference", + choices=("unknown", "wgs84-ellipsoid", "orthometric", "nap"), + default="unknown", ) plot.add_argument("--force", action="store_true", help="Replace an existing SVG") return root @@ -182,10 +182,10 @@ def main(argv: list[str] | None = None) -> int: local_map=args.map_data, terrain=args.terrain, terrain_reference=args.terrain_reference, - geoid=args.geoid, - confirm_geometric_ellipsoid=args.confirm_geometric_ellipsoid, force=args.force, refresh_map=args.refresh_map, + no_terrain=args.no_terrain, + refresh_terrain=args.refresh_terrain, ) print(f"Wrote {args.output} with {count} flight track(s).") except (ValueError, RuntimeError, OSError, sqlite3.Error) as exc: diff --git a/flight_map/maps.py b/flight_map/maps.py index d679853..c76d9b3 100644 --- a/flight_map/maps.py +++ b/flight_map/maps.py @@ -8,7 +8,9 @@ from pathlib import Path from typing import Any from urllib.parse import urlencode -from shapely.geometry import LineString +from shapely.geometry import GeometryCollection, LineString, Polygon +from shapely.geometry.base import BaseGeometry +from shapely.ops import polygonize_full, unary_union from flight_map.geo import Bounds, Projection from flight_map.network import request_json @@ -17,6 +19,10 @@ OVERPASS_URL = "https://overpass-api.de/api/interpreter" LOGGER = logging.getLogger(__name__) +class WaterMaskError(ValueError): + """Water geometry cannot reliably distinguish land from water.""" + + def validate_map_data(data: Any) -> dict[str, Any]: if not isinstance(data, dict) or not isinstance(data.get("elements"), list): raise ValueError("Map data must be an Overpass JSON object containing elements.") @@ -35,6 +41,8 @@ def map_data( "[out:json][timeout:60];(" f'way["highway"]({bbox});way["waterway"]({bbox});' f'way["building"]({bbox});way["natural"="water"]({bbox});' + f'way["waterway"="riverbank"]({bbox});' + f'relation["natural"="water"]({bbox});relation["waterway"="riverbank"]({bbox});' ");out geom;" ) key = hashlib.sha256((OVERPASS_URL + "\n" + query).encode()).hexdigest() @@ -77,6 +85,75 @@ def map_data( return validate_map_data(data) +def positions(geometry: Any) -> list[tuple[float, float]]: + if not isinstance(geometry, list) or len(geometry) < 2: + raise ValueError("Map way has no usable geometry; export using 'out geom'.") + try: + result = [(float(p["lon"]), float(p["lat"])) for p in geometry] + except (KeyError, ValueError, TypeError) as exc: + raise ValueError("Map way contains invalid geographic coordinates.") from exc + if any( + not math.isfinite(lon) + or not math.isfinite(lat) + or not -180 <= lon <= 180 + or not -90 <= lat <= 90 + for lon, lat in result + ): + raise ValueError("Map way coordinates must be finite and within geographic limits.") + return result + + +def water_areas(data: dict[str, Any]) -> BaseGeometry: + """Assemble closed OSM water areas, retaining islands in multipolygon relations.""" + areas: list[BaseGeometry] = [] + member_ids = set() + for element in data["elements"]: + if not isinstance(element, dict): + raise ValueError("Map data contains a malformed feature.") + tags = element.get("tags", {}) + if not isinstance(tags, dict): + raise ValueError("Map feature tags must be an object.") + is_water = tags.get("natural") == "water" or tags.get("waterway") == "riverbank" + if element.get("type") != "relation" or not is_water: + continue + rings: dict[str, list[LineString]] = {"outer": [], "inner": []} + for member in element.get("members", []): + if member.get("type") != "way": + continue + role = member.get("role", "") or "outer" + if role in rings: + rings[role].append(LineString(positions(member.get("geometry")))) + member_ids.add(member["ref"]) + combined = {} + for role, segments in rings.items(): + polygons, cuts, dangles, invalid = polygonize_full(segments) + if not cuts.is_empty or not dangles.is_empty or not invalid.is_empty: + raise WaterMaskError("Water multipolygon has incomplete rings; cannot mask land.") + combined[role] = unary_union(list(polygons.geoms)) + if combined["outer"].is_empty: + raise WaterMaskError("Water multipolygon has no outer ring.") + areas.append(combined["outer"].difference(combined["inner"])) + for element in data["elements"]: + tags = element.get("tags", {}) + if ( + element.get("type") != "way" + or element.get("id") in member_ids + or not (tags.get("natural") == "water" or tags.get("waterway") == "riverbank") + ): + continue + points = positions(element.get("geometry")) + if len(points) < 4 or points[0] != points[-1]: + raise WaterMaskError( + f"Water way {element.get('id', 'unknown')} has an unclosed outline; " + "cannot mask land." + ) + polygon = Polygon(points) + if not polygon.is_valid: + raise WaterMaskError("Water area has invalid geometry; cannot mask land.") + areas.append(polygon) + return unary_union(areas) if areas else GeometryCollection() + + def map_lines( data: dict[str, Any], projection: Projection, *, min_building_size: float = 0 ) -> list[LineString]: @@ -100,22 +177,7 @@ def map_lines( "proposed", }: continue - geometry = element.get("geometry") - if not isinstance(geometry, list) or len(geometry) < 2: - raise ValueError("Map way has no usable geometry; export using 'out geom'.") - try: - positions = [(float(p["lon"]), float(p["lat"])) for p in geometry] - except (KeyError, ValueError, TypeError) as exc: - raise ValueError("Map way contains invalid geographic coordinates.") from exc - if any( - not math.isfinite(lon) - or not math.isfinite(lat) - or not -180 <= lon <= 180 - or not -90 <= lat <= 90 - for lon, lat in positions - ): - raise ValueError("Map way coordinates must be finite and within geographic limits.") - points = [projection.point(lon, lat) for lon, lat in positions] + points = [projection.point(lon, lat) for lon, lat in positions(element.get("geometry"))] line = LineString(points) if "building" in tags: west, south, east, north = line.bounds diff --git a/flight_map/network.py b/flight_map/network.py index 038c689..7eac5e8 100644 --- a/flight_map/network.py +++ b/flight_map/network.py @@ -6,6 +6,18 @@ from urllib.request import Request, urlopen USER_AGENT = "FlightMap/0.1 (local non-commercial flight mapping)" +def request_bytes(url: str, *, max_bytes: int = 32 * 1024 * 1024) -> bytes: + request = Request(url, headers={"User-Agent": USER_AGENT}) + try: + with urlopen(request, timeout=60) as response: + content = response.read(max_bytes + 1) + except (URLError, TimeoutError) as exc: + raise RuntimeError(f"Raster download failed: {exc}") from exc + if len(content) > max_bytes: + raise RuntimeError("Raster download exceeds the 32 MiB limit.") + return content + + def request_json( url: str, *, data: bytes | None = None, headers: dict[str, str] | None = None ) -> Any: diff --git a/flight_map/plot.py b/flight_map/plot.py index f2d9ae8..95c5211 100644 --- a/flight_map/plot.py +++ b/flight_map/plot.py @@ -12,10 +12,11 @@ from shapely.affinity import affine_transform from shapely.geometry import LineString, box from shapely.geometry.base import BaseGeometry +from flight_map.ahn import terrain_data from flight_map.curves import Cubic from flight_map.geo import Projection from flight_map.lettering import lettering -from flight_map.maps import map_data, map_lines +from flight_map.maps import WaterMaskError, map_data, map_lines, water_areas from flight_map.storage import database, get_session, load_observations from flight_map.terrain import Elevation from flight_map.tracks import make_tracks @@ -119,18 +120,24 @@ def render( local_map: Path | None = None, terrain: Path | None = None, terrain_reference: str = "unknown", - geoid: Path | None = None, - confirm_geometric_ellipsoid: bool = False, force: bool = False, refresh_map: bool = False, + no_terrain: bool = False, + refresh_terrain: bool = False, ) -> int: options = options or PlotOptions() options.validate() + if no_terrain and (terrain is not None or refresh_terrain): + raise ValueError("--no-terrain cannot be combined with --terrain or --refresh-terrain.") + if terrain is not None and refresh_terrain: + raise ValueError("--refresh-terrain applies to automatic AHN data, not --terrain.") + if terrain is None and terrain_reference != "unknown": + raise ValueError("--terrain-reference requires a local --terrain raster.") if output.exists() and not force: raise ValueError(f"Output already exists: {output}. Use --force to replace it.") if any( output.resolve() == source.resolve() - for source in (db, local_map, terrain, geoid) + for source in (db, local_map, terrain) if source is not None ): raise ValueError("SVG output must not overwrite the recording database or input data.") @@ -150,26 +157,45 @@ def render( if not observations: raise ValueError("No recorded observations in the selected interval.") projection = Projection(session.bounds) - with Elevation(terrain, terrain_reference, geoid, confirm_geometric_ellipsoid) as elevation: - ground = elevation.mean_ground(session.bounds) - tracks = make_tracks(observations, projection, elevation, ground, options.max_gap) + data = map_data(session.bounds, cache, local_map, refresh=refresh_map) + try: + water = water_areas(data) + except WaterMaskError as exc: + LOGGER.warning( + "Land elevation unavailable: %s. Reported flight altitudes are unaffected.", exc + ) + water = None + terrain_source = "disabled" if no_terrain else "local terrain" + if terrain is None and not no_terrain: + terrain = terrain_data(session.bounds, cache / "terrain", refresh=refresh_terrain) + terrain_reference = "nap" if terrain else "unknown" + terrain_source = "PDOK AHN DTM dtm_05m; NAP; CC0" + if terrain is not None and terrain.resolve() == output.resolve(): + raise ValueError("SVG output must not overwrite cached terrain data.") + with Elevation(terrain, terrain_reference) as elevation: + ground = elevation.mean_ground(session.bounds, water=water) if water is not None else None + land_coverage = elevation.land_coverage + land_samples = elevation.land_samples + valid_land_samples = elevation.valid_land_samples + water_samples = elevation.water_samples + tracks = make_tracks(observations, projection, options.max_gap) if not tracks: raise ValueError("No usable airborne tracks cross the map in the selected interval.") if ground is None: LOGGER.warning( - "Average ground elevation unavailable: supply a complete terrain raster " - "and its vertical reference." + "Average land elevation unavailable: no usable height map or land coverage. " + "Reported flight altitudes are unaffected." ) - missing = sum(track.average_height is None for track in tracks) + missing = sum(track.average_altitude is None for track in tracks) if missing: LOGGER.warning( - "%d track height(s) unavailable: compatible geometric altitude, terrain " - "and vertical-reference conversion are required.", + "%d track altitude(s) unavailable: reported barometric altitude is missing " + "on a contributing segment; no geometric-altitude fallback is used.", missing, ) - data = map_data(session.bounds, cache, local_map, refresh=refresh_map) row_height = options.font_height * 1.8 - legend_height = (len(tracks) + 4) * row_height + terrain_attribution = terrain_source.startswith("PDOK") + legend_height = (len(tracks) + 5 + int(terrain_attribution)) * row_height available_width = options.width - 2 * (options.margin + options.extension) available_height = options.height - 2 * (options.margin + options.extension) - legend_height if available_height < 25: @@ -205,11 +231,15 @@ def render( ET.SubElement(root, tag("desc")).text = ( f"Session {session.id}; bounds W,S,E,N={session.bounds.values}; " f"{utc(start)} to {utc(end)} UTC. Local equirectangular projection. " - "Height is time-weighted aircraft altitude within the map minus the " - "area-sampled mean ground elevation, not terrain clearance. " + "Flight altitude is the time-weighted mean reported barometric pressure altitude " + "within the map, in metres, without terrain subtraction or vertical corrections. " + "It is not height above ground or terrain clearance. " + "Mean land elevation is a separate summary excluding mapped OSM water. " + f"Terrain source: {terrain_source}; " f"Terrain reference: {terrain_reference}; " + f"valid land samples: {valid_land_samples}/{land_samples}; " + f"land coverage: {100 * land_coverage:.1f}%; excluded water samples: {water_samples}; " f"mean ground elevation in metres: {ground}; " - f"geometric ellipsoid interpretation confirmed: {confirm_geometric_ellipsoid}. " "Map data: OpenStreetMap contributors, ODbL, https://www.openstreetmap.org/copyright. " "Flight data: The OpenSky Network, https://opensky-network.org. " "Schaefer et al., Bringing Up OpenSky: A Large-scale ADS-B Sensor Network " @@ -245,9 +275,10 @@ def render( "data-callsign": track.observations[0].callsign or "unavailable", "data-start-utc": utc(track.start), "data-end-utc": utc(track.end), - "data-average-height-metres": ( - f"{track.average_height:.3f}" - if track.average_height is not None + "data-altitude-source": "barometric", + "data-average-barometric-altitude-metres": ( + f"{track.average_altitude:.3f}" + if track.average_altitude is not None else "unavailable" ), }, @@ -282,15 +313,18 @@ def render( legend_y = top + map_height + options.extension + row_height ground_label = "N/A" if ground is None else f"{ground:.1f} M" legend = [ - "FLIGHT TIMES UTC - HEIGHT ABOVE MAP AVERAGE GROUND, NOT TERRAIN CLEARANCE", - f"MAP AVERAGE GROUND ELEVATION: {ground_label} - N/A MEANS UNAVAILABLE", + "FLIGHT TIMES UTC - REPORTED BAROMETRIC ALTITUDE, NOT HEIGHT ABOVE GROUND", + f"MAP AVERAGE LAND ELEVATION: {ground_label} - N/A MEANS UNAVAILABLE", + f"LAND ELEVATION COVERAGE: {100 * land_coverage:.1f} PERCENT - MAPPED WATER EXCLUDED", ] for index, track in enumerate(tracks, 1): first = track.observations[0] - height_label = "N/A" if track.average_height is None else f"{track.average_height:.0f} M" + altitude_label = ( + "N/A" if track.average_altitude is None else f"{track.average_altitude:.0f} M" + ) legend.append( f"{index}: {utc(track.start)} TO {utc(track.end)} " - f"{first.callsign or 'CALLSIGN N/A'} ({first.icao24.upper()}) HEIGHT {height_label}" + f"{first.callsign or 'CALLSIGN N/A'} ({first.icao24.upper()}) BARO ALT {altitude_label}" ) legend.extend( [ @@ -298,6 +332,8 @@ def render( "FLIGHTS: THE OPENSKY NETWORK - OPENSKY-NETWORK.ORG", ] ) + if terrain_attribution: + legend.append("TERRAIN: AHN DTM VIA PDOK (CC0) - ELEVATIONS RELATIVE TO NAP") for index, text in enumerate(legend): if len(text) * options.font_height > options.width - 2 * options.margin: raise ValueError("Legend is too wide; reduce --font-height or increase page width.") diff --git a/flight_map/terrain.py b/flight_map/terrain.py index c5932d8..24f5ee9 100644 --- a/flight_map/terrain.py +++ b/flight_map/terrain.py @@ -1,3 +1,4 @@ +import logging import math from contextlib import ExitStack from pathlib import Path @@ -5,39 +6,40 @@ from typing import Any import rasterio from pyproj import Transformer +from shapely.geometry import Point +from shapely.geometry.base import BaseGeometry from flight_map.geo import Bounds -from flight_map.storage import Observation + +LOGGER = logging.getLogger(__name__) class Elevation: - """Metre-valued terrain and optional h-H geoid separation rasters.""" + """Metre-valued terrain for the separate map land-elevation summary.""" def __init__( self, terrain: Path | None, reference: str = "unknown", - geoid: Path | None = None, - confirm_geometric_ellipsoid: bool = False, ) -> None: self.terrain_path = terrain self.reference = reference - self.geoid_path = geoid - self.confirm_geometric_ellipsoid = confirm_geometric_ellipsoid self.stack = ExitStack() self.ground: Any = None - self.geoid: Any = None self.ground_transform: Transformer | None = None - self.geoid_transform: Transformer | None = None + self.land_samples = 0 + self.valid_land_samples = 0 + self.water_samples = 0 + + @property + def land_coverage(self) -> float: + return self.valid_land_samples / self.land_samples if self.land_samples else 0.0 def __enter__(self) -> "Elevation": with ExitStack() as stack: if self.terrain_path: self.ground = stack.enter_context(rasterio.open(self.terrain_path)) self.ground_transform = self._transform(self.ground) - if self.geoid_path: - self.geoid = stack.enter_context(rasterio.open(self.geoid_path)) - self.geoid_transform = self._transform(self.geoid) self.stack = stack.pop_all() return self @@ -64,7 +66,10 @@ class Elevation: result = float(value) * dataset.scales[0] + dataset.offsets[0] return result if math.isfinite(result) else None - def mean_ground(self, bounds: Bounds, samples: int = 64) -> float | None: + def mean_ground( + self, bounds: Bounds, samples: int = 64, *, water: BaseGeometry | None = None + ) -> float | None: + self.land_samples = self.valid_land_samples = self.water_samples = 0 if not self.ground or self.ground_transform is None or self.reference == "unknown": return None if samples < 2: @@ -81,22 +86,35 @@ class Elevation: ) for col in range(samples): lon = bounds.west + (bounds.east - bounds.west) * (col + 0.5) / samples + if water is not None and water.covers(Point(lon, lat)): + self.water_samples += 1 + continue + self.land_samples += 1 value = self.sample(self.ground, self.ground_transform, lon, lat) if value is None: - return None + continue + self.valid_land_samples += 1 values.append(value) - return sum(values) / len(values) - - def aircraft_altitude(self, observation: Observation) -> float | None: - if not self.confirm_geometric_ellipsoid or observation.geo_altitude is None: + if not values: + LOGGER.warning( + "Average land elevation unavailable: no valid land samples " + "(%d mapped water samples excluded).", + self.water_samples, + ) return None - if self.reference == "wgs84-ellipsoid": - return observation.geo_altitude - if self.reference != "orthometric" or not self.geoid or self.geoid_transform is None: - return None - separation = self.sample( - self.geoid, self.geoid_transform, observation.longitude, observation.latitude + if self.valid_land_samples < self.land_samples: + LOGGER.warning( + "Incomplete height map: %.1f%% of sampled land has elevation data " + "(%d/%d); averaging valid land only. Missing land is not treated as water.", + 100 * self.land_coverage, + self.valid_land_samples, + self.land_samples, + ) + LOGGER.info( + "Land elevation coverage: %.1f%% (%d/%d); %d water samples excluded.", + 100 * self.land_coverage, + self.valid_land_samples, + self.land_samples, + self.water_samples, ) - if separation is None: - return None - return observation.geo_altitude - separation + return sum(values) / len(values) diff --git a/flight_map/tracks.py b/flight_map/tracks.py index cfb8396..f31080b 100644 --- a/flight_map/tracks.py +++ b/flight_map/tracks.py @@ -5,14 +5,13 @@ from itertools import groupby, pairwise from flight_map.curves import Cubic, interpolate from flight_map.geo import Projection from flight_map.storage import Observation -from flight_map.terrain import Elevation @dataclass class Track: observations: list[Observation] curves: list[Cubic] - average_height: float | None + average_altitude: float | None start: float end: float @@ -48,8 +47,6 @@ def split_tracks(observations: list[Observation], max_gap: float) -> list[list[O def make_tracks( observations: list[Observation], projection: Projection, - elevation: Elevation, - ground: float | None, max_gap: float, ) -> list[Track]: result = [] @@ -60,18 +57,18 @@ def make_tracks( continue total_time = 0.0 altitude_integral = 0.0 - missing_height = ground is None + missing_altitude = False start, end = math.inf, -math.inf for index, (first, last) in enumerate(pairwise(items)): duration = last.timestamp - first.timestamp - h0, h1 = elevation.aircraft_altitude(first), elevation.aircraft_altitude(last) + h0, h1 = first.baro_altitude, last.baro_altitude for f0, f1 in curves[index].inside_intervals(projection.rectangle.bounds): inside_duration = duration * (f1 - f0) total_time += inside_duration start = min(start, first.timestamp + f0 * duration) end = max(end, first.timestamp + f1 * duration) if h0 is None or h1 is None: - missing_height = True + missing_altitude = True else: entry = h0 + (h1 - h0) * f0 exit_height = h0 + (h1 - h0) * f1 @@ -79,7 +76,7 @@ def make_tracks( if total_time == 0: continue average = None - if not missing_height and ground is not None: - average = altitude_integral / total_time - ground + if not missing_altitude: + average = altitude_integral / total_time result.append(Track(items, curves, average, start, end)) return sorted(result, key=lambda track: (track.start, track.observations[0].icao24)) diff --git a/tests/test_ahn.py b/tests/test_ahn.py new file mode 100644 index 0000000..f11ec82 --- /dev/null +++ b/tests/test_ahn.py @@ -0,0 +1,305 @@ +import json +from pathlib import Path +from unittest.mock import Mock +from urllib.parse import parse_qs, urlparse +from xml.etree import ElementTree as ET + +import numpy as np +import pytest +import rasterio +from pyproj import Transformer +from rasterio.io import MemoryFile +from rasterio.transform import from_bounds +from shapely.geometry import box + +from flight_map.ahn import SIZE, coverage_url, terrain_data +from flight_map.geo import DEFAULT_BOUNDS, Bounds +from flight_map.maps import water_areas +from flight_map.network import request_bytes +from flight_map.plot import SVG_NS, render +from flight_map.storage import ( + Observation, + checkpoint, + create_session, + database, + initialize, + save_observations, +) +from flight_map.terrain import Elevation + +BOUNDS = DEFAULT_BOUNDS + + +def ahn_bytes(value: float = -3) -> bytes: + bounds = Transformer.from_crs(4326, 28992, always_xy=True).transform_bounds(*BOUNDS.values) + with MemoryFile() as memory: + with memory.open( + driver="GTiff", + width=SIZE, + height=SIZE, + count=1, + dtype="float32", + crs="EPSG:28992", + transform=from_bounds(*bounds, SIZE, SIZE), + nodata=-9999, + ) as dataset: + dataset.write(np.full((SIZE, SIZE), value, dtype="float32"), 1) + return memory.read() + + +def test_pdok_request_cache_refresh_and_metadata( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + fetch = Mock(return_value=ahn_bytes()) + monkeypatch.setattr("flight_map.ahn.request_bytes", fetch) + path = terrain_data(BOUNDS, tmp_path) + assert path is not None + assert terrain_data(BOUNDS, tmp_path) == path + fetch.assert_called_once() + params = parse_qs(urlparse(fetch.call_args.args[0]).query) + assert params["coverage"] == ["dtm_05m"] + assert params["crs"] == ["EPSG:28992"] + assert params["format"] == ["GEOTIFF"] + assert params["width"] == params["height"] == [str(SIZE)] + with rasterio.open(path) as dataset: + assert dataset.tags()["vertical_reference"] == "NAP" + assert dataset.tags()["license"] == "CC0" + assert terrain_data(BOUNDS, tmp_path, refresh=True) == path + assert fetch.call_count == 2 + assert coverage_url(BOUNDS) != coverage_url(Bounds(4.7, 52.2, 4.8, 52.3)) + before = path.read_bytes() + fetch.side_effect = RuntimeError("HTTP 503") + assert terrain_data(BOUNDS, tmp_path, refresh=True) is None + assert path.read_bytes() == before + assert len(list(tmp_path.glob("*.tif"))) == 1 + + +def test_unavailable_corrupt_and_outside_coverage_warn( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + fetch = Mock(return_value=b"no coverage") + monkeypatch.setattr("flight_map.ahn.request_bytes", fetch) + assert terrain_data(BOUNDS, tmp_path) is None + assert "height map unavailable" in caplog.text + assert not list(tmp_path.iterdir()) + fetch.reset_mock() + assert terrain_data(Bounds(-1, 50, 0, 51), tmp_path) is None + fetch.assert_not_called() + assert "outside Dutch AHN coverage" in caplog.text + fetch.return_value = ahn_bytes() + path = terrain_data(BOUNDS, tmp_path) + assert path is not None + path.write_bytes(b"broken") + assert terrain_data(BOUNDS, tmp_path) is None + assert "refresh-terrain" in caplog.text + assert terrain_data(BOUNDS, tmp_path, refresh=True) == path + + +def test_failed_cache_replace_keeps_old_height_map( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("flight_map.ahn.request_bytes", Mock(return_value=ahn_bytes())) + path = terrain_data(BOUNDS, tmp_path) + assert path is not None + before = path.read_bytes() + monkeypatch.setattr("flight_map.ahn.os.replace", Mock(side_effect=OSError("disk error"))) + assert terrain_data(BOUNDS, tmp_path, refresh=True) is None + assert path.read_bytes() == before + assert list(tmp_path.iterdir()) == [path] + + +def water_way( + points: list[tuple[float, float]], + identifier: int = 1, +) -> dict[str, object]: + return { + "type": "way", + "id": identifier, + "tags": {"natural": "water"}, + "geometry": [{"lon": x, "lat": y} for x, y in points], + } + + +def test_water_polygon_and_relation_preserve_islands() -> None: + outer = [(0, 0), (4, 0), (4, 4), (0, 4), (0, 0)] + inner = [(1, 1), (3, 1), (3, 3), (1, 3), (1, 1)] + members = [ + {"type": "way", "ref": 1, "role": "outer", "geometry": water_way(outer)["geometry"]}, + {"type": "way", "ref": 2, "role": "inner", "geometry": water_way(inner)["geometry"]}, + ] + relation = { + "type": "relation", + "tags": {"natural": "water", "type": "multipolygon"}, + "members": members, + } + geometry = water_areas({"elements": [water_way(outer), relation]}) + assert geometry.area == 12 + assert not geometry.covers(box(1.5, 1.5, 2.5, 2.5)) + assert water_areas({"elements": [water_way(outer)]}).area == 16 + # Relations often consist of several open ways joined into an outer ring. + members[0]["geometry"] = water_way(outer[:3])["geometry"] + members.append( + {"type": "way", "ref": 3, "role": "outer", "geometry": water_way(outer[2:])["geometry"]} + ) + assert water_areas({"elements": [relation]}).area == 12 + assert water_areas({"elements": []}).is_empty + with pytest.raises(ValueError, match="unclosed"): + water_areas({"elements": [water_way(outer[:3])]}) + members.pop() + with pytest.raises(ValueError, match="incomplete"): + water_areas({"elements": [relation]}) + + +def test_land_mean_excludes_water_but_reports_missing_land( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + path = tmp_path / "terrain.tif" + bounds = Bounds(0, 0, 4, 4) + values = np.full((4, 4), 100, dtype="float32") + values[:, :2] = -50 # These are valid water elevations, which must not enter the mean. + values[0, 2] = -9999 # Missing land is a separate coverage issue. + with rasterio.open( + path, + "w", + driver="GTiff", + width=4, + height=4, + count=1, + dtype="float32", + crs="EPSG:4326", + transform=from_bounds(*bounds.values, 4, 4), + nodata=-9999, + ) as dataset: + dataset.write(values, 1) + with Elevation(path, "nap") as elevation: + assert elevation.mean_ground(bounds, 4, water=box(0, 0, 2, 4)) == 100 + assert elevation.water_samples == 8 + assert elevation.land_samples == 8 + assert elevation.valid_land_samples == 7 + assert elevation.land_coverage == 7 / 8 + assert "87.5%" in caplog.text + assert "Missing land is not treated as water" in caplog.text + assert elevation.mean_ground(bounds, 4, water=box(0, 0, 4, 4)) is None + + +def test_plot_auto_ahn_and_warning_fallback( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + db = tmp_path / "flights.sqlite" + with database(db) as connection: + initialize(connection) + session = create_session(connection, BOUNDS, 2000, 30, 100) + save_observations( + connection, + session, + [ + Observation("abc123", "TEST", 110, 4.79, 52.24, 900, 1043), + Observation("abc123", "TEST", 140, 4.85, 52.24, 900, 1043), + ], + 150, + ) + checkpoint(connection, session.id, 150, "stopped") + local = tmp_path / "map.json" + local.write_text( + json.dumps( + { + "elements": [ + { + "type": "way", + "geometry": [{"lon": 4.79, "lat": 52.24}, {"lon": 4.85, "lat": 52.24}], + }, + ] + } + ) + ) + monkeypatch.setattr("flight_map.ahn.request_bytes", Mock(return_value=ahn_bytes(-3))) + output = tmp_path / "map.svg" + assert render(db, session.id, output, local_map=local, cache=tmp_path / "cache") == 1 + text = output.read_text() + assert "PDOK AHN DTM" in text and "Terrain reference: nap" in text + root = ET.parse(output).getroot() + flight = root.find(f".//{{{SVG_NS}}}g[@id='flight-1']") + assert flight is not None + assert flight.attrib["data-average-barometric-altitude-metres"] == "900.000" + assert "altitude(s) unavailable" not in caplog.text + assert "ground elevation in metres: -3.0" in text + monkeypatch.setattr("flight_map.ahn.request_bytes", Mock(return_value=ahn_bytes(100))) + changed_output = tmp_path / "changed-ground.svg" + assert ( + render( + db, + session.id, + changed_output, + local_map=local, + cache=tmp_path / "cache", + refresh_terrain=True, + ) + == 1 + ) + changed_flight = ET.parse(changed_output).getroot().find(f".//{{{SVG_NS}}}g[@id='flight-1']") + assert changed_flight is not None + assert changed_flight.attrib["data-average-barometric-altitude-metres"] == "900.000" + assert "ground elevation in metres: 100.0" in changed_output.read_text() + monkeypatch.setattr("flight_map.ahn.request_bytes", Mock(side_effect=RuntimeError("offline"))) + assert ( + render( + db, + session.id, + tmp_path / "unavailable.svg", + local_map=local, + cache=tmp_path / "other-cache", + ) + == 1 + ) + assert "AHN height map unavailable" in caplog.text + assert "Average land elevation unavailable" in caplog.text + unavailable = ET.parse(tmp_path / "unavailable.svg").getroot() + unchanged = unavailable.find(f".//{{{SVG_NS}}}g[@id='flight-1']") + assert unchanged is not None + assert unchanged.attrib["data-average-barometric-altitude-metres"] == "900.000" + invalid_water = json.loads(local.read_text()) + invalid_water["elements"].append(water_way([(4.80, 52.24), (4.81, 52.25)])) + local.write_text(json.dumps(invalid_water)) + assert render(db, session.id, tmp_path / "bad-water.svg", local_map=local, no_terrain=True) == 1 + assert "unclosed outline" in caplog.text + assert "Reported flight altitudes are unaffected" in caplog.text + downloader = Mock(side_effect=AssertionError("must stay offline")) + monkeypatch.setattr("flight_map.plot.terrain_data", downloader) + assert render(db, session.id, tmp_path / "disabled.svg", local_map=local, no_terrain=True) == 1 + downloader.assert_not_called() + with database(db) as connection: + with connection: + connection.execute("UPDATE observations SET baro_altitude = NULL") + missing_output = tmp_path / "missing-baro.svg" + assert render(db, session.id, missing_output, local_map=local, no_terrain=True) == 1 + missing_flight = ET.parse(missing_output).getroot().find(f".//{{{SVG_NS}}}g[@id='flight-1']") + assert missing_flight is not None + assert missing_flight.attrib["data-average-barometric-altitude-metres"] == "unavailable" + assert "reported barometric altitude is missing" in caplog.text + assert "no geometric-altitude fallback" in caplog.text + with pytest.raises(ValueError, match="cannot be combined"): + render(db, session.id, tmp_path / "bad.svg", no_terrain=True, refresh_terrain=True) + + +def test_binary_download_errors_and_size_limit(monkeypatch: pytest.MonkeyPatch) -> None: + from urllib.error import URLError + + response = Mock() + response.__enter__ = Mock(return_value=response) + response.__exit__ = Mock(return_value=False) + response.read.return_value = b"12345" + fetch = Mock(return_value=response) + monkeypatch.setattr("flight_map.network.urlopen", fetch) + assert request_bytes("https://example.invalid") == b"12345" + with pytest.raises(RuntimeError, match="limit"): + request_bytes("https://example.invalid", max_bytes=4) + fetch.side_effect = URLError("offline") + with pytest.raises(RuntimeError, match="download failed"): + request_bytes("https://example.invalid") diff --git a/tests/test_flight_map.py b/tests/test_flight_map.py index 76cbd4b..74aba6c 100644 --- a/tests/test_flight_map.py +++ b/tests/test_flight_map.py @@ -105,8 +105,8 @@ def local_map(tmp_path: Path) -> Path: def test_default_and_reversed_bounds() -> None: - assert DEFAULT_BOUNDS.values == (4.784498, 52.232025, 4.863525, 52.253985) - assert Bounds.corners(4.863525, 52.253985, 4.784498, 52.232025) == DEFAULT_BOUNDS + assert DEFAULT_BOUNDS.values == (4.784, 52.232, 4.864, 52.258) + assert Bounds.corners(4.864, 52.258, 4.784, 52.232) == DEFAULT_BOUNDS assert DEFAULT_BOUNDS.contains(4.83, 52.24) expanded = DEFAULT_BOUNDS.expand(2000) assert expanded.west < DEFAULT_BOUNDS.west and expanded.north > DEFAULT_BOUNDS.north @@ -174,29 +174,22 @@ def test_database_readonly(recorded: tuple[Path, Session]) -> None: connection.execute("DELETE FROM observations") -def test_terrain_and_datum_conversion(tmp_path: Path) -> None: +def test_local_terrain_summary(tmp_path: Path) -> None: terrain = raster(tmp_path / "terrain.tif", -2) - geoid = raster(tmp_path / "geoid.tif", 43) - with Elevation(terrain, "orthometric", geoid, True) as elevation: + with Elevation(terrain, "orthometric") as elevation: assert elevation.mean_ground(DEFAULT_BOUNDS, 4) == -2 - assert elevation.aircraft_altitude(observation()) == 1000 - with Elevation(terrain, "orthometric", geoid, False) as elevation: - assert elevation.aircraft_altitude(observation()) is None - with Elevation(terrain, "orthometric", None, True) as elevation: - assert elevation.aircraft_altitude(observation()) is None - with Elevation(terrain, "wgs84-ellipsoid", None, True) as elevation: - assert elevation.aircraft_altitude(observation()) == 1043 - assert elevation.aircraft_altitude(observation(geo_altitude=None)) is None -def test_terrain_missing_data(tmp_path: Path) -> None: +def test_terrain_missing_data(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: terrain = raster(tmp_path / "terrain.tif", -9999) with Elevation(terrain, "wgs84-ellipsoid") as elevation: assert elevation.mean_ground(DEFAULT_BOUNDS, 4) is None small = Bounds(4.80, 52.24, 4.81, 52.25) raster(terrain, 0, bounds=small) with Elevation(terrain, "wgs84-ellipsoid") as elevation: - assert elevation.mean_ground(DEFAULT_BOUNDS, 4) is None + assert elevation.mean_ground(DEFAULT_BOUNDS, 16) == 0 + assert 0 < elevation.land_coverage < 1 + assert "averaging valid land only" in caplog.text def test_equal_area_ground_average(tmp_path: Path) -> None: @@ -209,18 +202,17 @@ def test_equal_area_ground_average(tmp_path: Path) -> None: assert elevation.mean_ground(DEFAULT_BOUNDS, 10) == pytest.approx(5) -def test_height_is_time_weighted_and_excludes_extensions() -> None: +def test_reported_altitude_is_time_weighted_and_excludes_extensions() -> None: bounds = Bounds(0, 0, 2, 1) observations = [ - observation(longitude=-1, latitude=0.5, timestamp=T0, geo_altitude=0), - observation(longitude=1, latitude=0.5, timestamp=T0 + 20, geo_altitude=200), - observation(longitude=3, latitude=0.5, timestamp=T0 + 60, geo_altitude=600), + observation(longitude=-1, latitude=0.5, timestamp=T0, baro_altitude=0), + observation(longitude=1, latitude=0.5, timestamp=T0 + 20, baro_altitude=200), + observation(longitude=3, latitude=0.5, timestamp=T0 + 60, baro_altitude=600), ] - with Elevation(None, "wgs84-ellipsoid", None, True) as elevation: - tracks = make_tracks(observations, Projection(bounds), elevation, 10, 120) + tracks = make_tracks(observations, Projection(bounds), 120) # Inside: 10s at mean altitude 150 + 20s at mean altitude 300. assert len(tracks) == 1 - assert tracks[0].average_height == pytest.approx(240) + assert tracks[0].average_altitude == pytest.approx(250) assert tracks[0].start == pytest.approx(T0 + 10) assert tracks[0].end == pytest.approx(T0 + 40) @@ -229,13 +221,31 @@ def test_stationary_positions_and_missing_altitude() -> None: items = [ observation(), observation(timestamp=T0 + 25), + observation(timestamp=T0 + 40, longitude=4.82, baro_altitude=None), + ] + tracks = make_tracks(items, Projection(DEFAULT_BOUNDS), 120) + assert len(tracks) == 1 and tracks[0].average_altitude is None + assert make_tracks(items[:2], Projection(DEFAULT_BOUNDS), 120) == [] + + +def test_reported_altitude_does_not_require_geometric_altitude() -> None: + items = [ + observation(geo_altitude=None), observation(timestamp=T0 + 40, longitude=4.82, geo_altitude=None), ] - with Elevation(None, "wgs84-ellipsoid", None, True) as elevation: - tracks = make_tracks(items, Projection(DEFAULT_BOUNDS), elevation, -2, 120) - assert len(tracks) == 1 and tracks[0].average_height is None - with Elevation(None) as elevation: - assert make_tracks(items[:2], Projection(DEFAULT_BOUNDS), elevation, None, 120) == [] + tracks = make_tracks(items, Projection(DEFAULT_BOUNDS), 120) + assert len(tracks) == 1 + assert tracks[0].average_altitude == 900 + + +@pytest.mark.parametrize("altitude", [0, -50, 1234]) +def test_zero_negative_and_positive_reported_altitudes_are_preserved(altitude: float) -> None: + items = [ + observation(baro_altitude=altitude, geo_altitude=9999), + observation(timestamp=T0 + 40, longitude=4.82, baro_altitude=altitude, geo_altitude=9999), + ] + tracks = make_tracks(items, Projection(DEFAULT_BOUNDS), 120) + assert tracks[0].average_altitude == pytest.approx(altitude) def test_track_splitting_and_ground_filter() -> None: @@ -275,7 +285,6 @@ def test_end_to_end_offline_repeatable( local_map=local_map, terrain=terrain, terrain_reference="wgs84-ellipsoid", - confirm_geometric_ellipsoid=True, ) == 1 ) @@ -296,12 +305,13 @@ def test_end_to_end_offline_repeatable( assert flight_paths and all("C " in element.attrib["d"] for element in flight_paths) flight = root.find(f".//{{{SVG_NS}}}g[@id='flight-1']") assert flight is not None and flight.attrib["data-callsign"] == "KLM123" - assert flight.attrib["data-average-height-metres"] != "unavailable" + assert flight.attrib["data-average-barometric-altitude-metres"] == "900.000" + assert flight.attrib["data-altitude-source"] == "barometric" for element in root.iter(f"{{{SVG_NS}}}path"): assert "clip-path" not in element.attrib for x, y in re.findall(r"(-?\d+\.\d+),(-?\d+\.\d+)", element.attrib["d"]): assert 0 <= float(x) <= 297 and 0 <= float(y) <= 210 - assert "HEIGHT" not in output.read_text() # Lettering is geometry, not SVG text. + assert "BARO ALT" not in output.read_text() # Lettering is geometry, not SVG text. with pytest.raises(ValueError, match="already exists"): render(path, session.id, output, local_map=local_map) @@ -448,7 +458,7 @@ def test_opensky_parsing_and_auth(monkeypatch: pytest.MonkeyPatch) -> None: client = OpenSky() items = client.observations(DEFAULT_BOUNDS) assert items[0].callsign == "KLM123" and items[0].geo_altitude == 1043 - assert "lamin=52.232025" in fetch.call_args.args[0] + assert f"lamin={DEFAULT_BOUNDS.south}" in fetch.call_args.args[0] state[5] = None assert client.observations(DEFAULT_BOUNDS) == [] fetch.return_value = {"states": None} @@ -521,6 +531,7 @@ def test_cli_validation_and_plot( session.id, "--map-data", str(local_map), + "--no-terrain", "--output", str(tmp_path / "cli.svg"), ]