height correction, new defalt

This commit is contained in:
Olaf committed 2026-10-21 09:24:12 +02:00
1 parent 0891cf12a5
commit b692f94122
12 files changed
+792 -205

No files matched your search

+108
View File
@@ -0,0 +1,108 @@
"""PDOK AHN DTM coverage in RD New, elevations in metres relative to NAP."""
import hashlib
import logging
import math
import os
import tempfile
from pathlib import Path
from urllib.parse import urlencode
import rasterio
from pyproj import Transformer
from rasterio.errors import RasterioError
from flight_map.geo import Bounds
from flight_map.network import request_bytes
LOGGER = logging.getLogger(__name__)
WCS_URL = "https://service.pdok.nl/rws/ahn/wcs/v1_0"
COVERAGE = "dtm_05m"
AHN_EXTENT = Bounds(3.20030320333955, 50.728765211722, 7.273444085294, 53.555958411015)
SIZE = 512
def coverage_url(bounds: Bounds) -> str:
transform = Transformer.from_crs("EPSG:4326", "EPSG:28992", always_xy=True)
west, south, east, north = transform.transform_bounds(*bounds.values, densify_pts=21)
bbox = ",".join(
str(v) for v in (math.floor(west), math.floor(south), math.ceil(east), math.ceil(north))
)
return (
WCS_URL
+ "?"
+ urlencode(
{
"service": "WCS",
"version": "1.0.0",
"request": "GetCoverage",
"coverage": COVERAGE,
"crs": "EPSG:28992",
"response_crs": "EPSG:28992",
"bbox": bbox,
"width": SIZE,
"height": SIZE,
"format": "GEOTIFF",
"interpolation": "nearest neighbor",
}
)
)
def validate_raster(path: Path) -> None:
with rasterio.open(path) as dataset:
if dataset.driver != "GTiff" or dataset.crs != rasterio.crs.CRS.from_epsg(28992):
raise ValueError("AHN response is not an RD New GeoTIFF.")
if dataset.count != 1 or dataset.width != SIZE or dataset.height != SIZE:
raise ValueError("AHN response has unexpected raster dimensions.")
dataset.read(1, masked=True)
def terrain_data(bounds: Bounds, cache: Path, *, refresh: bool = False) -> Path | None:
if not (
AHN_EXTENT.contains(bounds.west, bounds.south)
and AHN_EXTENT.contains(bounds.east, bounds.north)
):
LOGGER.warning("AHN height map unavailable: map bounds extend outside Dutch AHN coverage.")
return None
url = coverage_url(bounds)
key = hashlib.sha256(url.encode()).hexdigest()
path = cache / f"ahn-{key}.tif"
temporary: Path | None = None
try:
if path.exists() and not refresh:
validate_raster(path)
LOGGER.info("Reading cached AHN DTM (NAP): %s", path)
return path
LOGGER.info("Downloading AHN DTM height map from PDOK for %s", bounds.values)
content = request_bytes(url)
cache.mkdir(parents=True, exist_ok=True)
with tempfile.NamedTemporaryFile(dir=cache, suffix=".tif", delete=False) as stream:
temporary = Path(stream.name)
stream.write(content)
stream.flush()
os.fsync(stream.fileno())
validate_raster(temporary)
with rasterio.open(temporary, "r+") as dataset:
dataset.update_tags(
source=WCS_URL,
coverage=COVERAGE,
vertical_reference="NAP",
units="metres",
license="CC0",
request_url=url,
)
os.replace(temporary, path)
LOGGER.info("Cached AHN DTM (NAP): %s", path)
return path
except (RuntimeError, OSError, ValueError, RasterioError) as exc:
LOGGER.warning(
"AHN height map unavailable: %s. Land elevation will be N/A; "
"reported flight altitudes are unaffected. "
"Use --refresh-terrain to retry a damaged cache entry.",
exc,
)
return None
finally:
if temporary and temporary.exists():
temporary.unlink()
+14 -14
View File
@@ -96,19 +96,19 @@ def parser() -> argparse.ArgumentParser:
"--terrain", type=Path, help="Local terrain GeoTIFF; elevations must be in metres"
)
plot.add_argument(
"--terrain-reference",
choices=("unknown", "wgs84-ellipsoid", "orthometric"),
default="unknown",
)
plot.add_argument(
"--geoid",
type=Path,
help="Metre-valued h-H raster matching the terrain's orthometric datum",
)
plot.add_argument(
"--confirm-geometric-ellipsoid",
"--no-terrain",
action="store_true",
help="Confirm OpenSky geometric heights can be treated as WGS84 ellipsoidal",
help="Disable the separate land-elevation summary and automatic AHN download",
)
plot.add_argument(
"--refresh-terrain",
action="store_true",
help="Refresh the cached PDOK AHN DTM for this map",
)
plot.add_argument(
"--terrain-reference",
choices=("unknown", "wgs84-ellipsoid", "orthometric", "nap"),
default="unknown",
)
plot.add_argument("--force", action="store_true", help="Replace an existing SVG")
return root
@@ -182,10 +182,10 @@ def main(argv: list[str] | None = None) -> int:
local_map=args.map_data,
terrain=args.terrain,
terrain_reference=args.terrain_reference,
geoid=args.geoid,
confirm_geometric_ellipsoid=args.confirm_geometric_ellipsoid,
force=args.force,
refresh_map=args.refresh_map,
no_terrain=args.no_terrain,
refresh_terrain=args.refresh_terrain,
)
print(f"Wrote {args.output} with {count} flight track(s).")
except (ValueError, RuntimeError, OSError, sqlite3.Error) as exc:
+79 -17
View File
@@ -8,7 +8,9 @@ from pathlib import Path
from typing import Any
from urllib.parse import urlencode
from shapely.geometry import LineString
from shapely.geometry import GeometryCollection, LineString, Polygon
from shapely.geometry.base import BaseGeometry
from shapely.ops import polygonize_full, unary_union
from flight_map.geo import Bounds, Projection
from flight_map.network import request_json
@@ -17,6 +19,10 @@ OVERPASS_URL = "https://overpass-api.de/api/interpreter"
LOGGER = logging.getLogger(__name__)
class WaterMaskError(ValueError):
"""Water geometry cannot reliably distinguish land from water."""
def validate_map_data(data: Any) -> dict[str, Any]:
if not isinstance(data, dict) or not isinstance(data.get("elements"), list):
raise ValueError("Map data must be an Overpass JSON object containing elements.")
@@ -35,6 +41,8 @@ def map_data(
"[out:json][timeout:60];("
f'way["highway"]({bbox});way["waterway"]({bbox});'
f'way["building"]({bbox});way["natural"="water"]({bbox});'
f'way["waterway"="riverbank"]({bbox});'
f'relation["natural"="water"]({bbox});relation["waterway"="riverbank"]({bbox});'
");out geom;"
)
key = hashlib.sha256((OVERPASS_URL + "\n" + query).encode()).hexdigest()
@@ -77,6 +85,75 @@ def map_data(
return validate_map_data(data)
def positions(geometry: Any) -> list[tuple[float, float]]:
if not isinstance(geometry, list) or len(geometry) < 2:
raise ValueError("Map way has no usable geometry; export using 'out geom'.")
try:
result = [(float(p["lon"]), float(p["lat"])) for p in geometry]
except (KeyError, ValueError, TypeError) as exc:
raise ValueError("Map way contains invalid geographic coordinates.") from exc
if any(
not math.isfinite(lon)
or not math.isfinite(lat)
or not -180 <= lon <= 180
or not -90 <= lat <= 90
for lon, lat in result
):
raise ValueError("Map way coordinates must be finite and within geographic limits.")
return result
def water_areas(data: dict[str, Any]) -> BaseGeometry:
"""Assemble closed OSM water areas, retaining islands in multipolygon relations."""
areas: list[BaseGeometry] = []
member_ids = set()
for element in data["elements"]:
if not isinstance(element, dict):
raise ValueError("Map data contains a malformed feature.")
tags = element.get("tags", {})
if not isinstance(tags, dict):
raise ValueError("Map feature tags must be an object.")
is_water = tags.get("natural") == "water" or tags.get("waterway") == "riverbank"
if element.get("type") != "relation" or not is_water:
continue
rings: dict[str, list[LineString]] = {"outer": [], "inner": []}
for member in element.get("members", []):
if member.get("type") != "way":
continue
role = member.get("role", "") or "outer"
if role in rings:
rings[role].append(LineString(positions(member.get("geometry"))))
member_ids.add(member["ref"])
combined = {}
for role, segments in rings.items():
polygons, cuts, dangles, invalid = polygonize_full(segments)
if not cuts.is_empty or not dangles.is_empty or not invalid.is_empty:
raise WaterMaskError("Water multipolygon has incomplete rings; cannot mask land.")
combined[role] = unary_union(list(polygons.geoms))
if combined["outer"].is_empty:
raise WaterMaskError("Water multipolygon has no outer ring.")
areas.append(combined["outer"].difference(combined["inner"]))
for element in data["elements"]:
tags = element.get("tags", {})
if (
element.get("type") != "way"
or element.get("id") in member_ids
or not (tags.get("natural") == "water" or tags.get("waterway") == "riverbank")
):
continue
points = positions(element.get("geometry"))
if len(points) < 4 or points[0] != points[-1]:
raise WaterMaskError(
f"Water way {element.get('id', 'unknown')} has an unclosed outline; "
"cannot mask land."
)
polygon = Polygon(points)
if not polygon.is_valid:
raise WaterMaskError("Water area has invalid geometry; cannot mask land.")
areas.append(polygon)
return unary_union(areas) if areas else GeometryCollection()
def map_lines(
data: dict[str, Any], projection: Projection, *, min_building_size: float = 0
) -> list[LineString]:
@@ -100,22 +177,7 @@ def map_lines(
"proposed",
}:
continue
geometry = element.get("geometry")
if not isinstance(geometry, list) or len(geometry) < 2:
raise ValueError("Map way has no usable geometry; export using 'out geom'.")
try:
positions = [(float(p["lon"]), float(p["lat"])) for p in geometry]
except (KeyError, ValueError, TypeError) as exc:
raise ValueError("Map way contains invalid geographic coordinates.") from exc
if any(
not math.isfinite(lon)
or not math.isfinite(lat)
or not -180 <= lon <= 180
or not -90 <= lat <= 90
for lon, lat in positions
):
raise ValueError("Map way coordinates must be finite and within geographic limits.")
points = [projection.point(lon, lat) for lon, lat in positions]
points = [projection.point(lon, lat) for lon, lat in positions(element.get("geometry"))]
line = LineString(points)
if "building" in tags:
west, south, east, north = line.bounds
+12
View File
@@ -6,6 +6,18 @@ from urllib.request import Request, urlopen
USER_AGENT = "FlightMap/0.1 (local non-commercial flight mapping)"
def request_bytes(url: str, *, max_bytes: int = 32 * 1024 * 1024) -> bytes:
request = Request(url, headers={"User-Agent": USER_AGENT})
try:
with urlopen(request, timeout=60) as response:
content = response.read(max_bytes + 1)
except (URLError, TimeoutError) as exc:
raise RuntimeError(f"Raster download failed: {exc}") from exc
if len(content) > max_bytes:
raise RuntimeError("Raster download exceeds the 32 MiB limit.")
return content
def request_json(
url: str, *, data: bytes | None = None, headers: dict[str, str] | None = None
) -> Any:
+60 -24
View File
@@ -12,10 +12,11 @@ from shapely.affinity import affine_transform
from shapely.geometry import LineString, box
from shapely.geometry.base import BaseGeometry
from flight_map.ahn import terrain_data
from flight_map.curves import Cubic
from flight_map.geo import Projection
from flight_map.lettering import lettering
from flight_map.maps import map_data, map_lines
from flight_map.maps import WaterMaskError, map_data, map_lines, water_areas
from flight_map.storage import database, get_session, load_observations
from flight_map.terrain import Elevation
from flight_map.tracks import make_tracks
@@ -119,18 +120,24 @@ def render(
local_map: Path | None = None,
terrain: Path | None = None,
terrain_reference: str = "unknown",
geoid: Path | None = None,
confirm_geometric_ellipsoid: bool = False,
force: bool = False,
refresh_map: bool = False,
no_terrain: bool = False,
refresh_terrain: bool = False,
) -> int:
options = options or PlotOptions()
options.validate()
if no_terrain and (terrain is not None or refresh_terrain):
raise ValueError("--no-terrain cannot be combined with --terrain or --refresh-terrain.")
if terrain is not None and refresh_terrain:
raise ValueError("--refresh-terrain applies to automatic AHN data, not --terrain.")
if terrain is None and terrain_reference != "unknown":
raise ValueError("--terrain-reference requires a local --terrain raster.")
if output.exists() and not force:
raise ValueError(f"Output already exists: {output}. Use --force to replace it.")
if any(
output.resolve() == source.resolve()
for source in (db, local_map, terrain, geoid)
for source in (db, local_map, terrain)
if source is not None
):
raise ValueError("SVG output must not overwrite the recording database or input data.")
@@ -150,26 +157,45 @@ def render(
if not observations:
raise ValueError("No recorded observations in the selected interval.")
projection = Projection(session.bounds)
with Elevation(terrain, terrain_reference, geoid, confirm_geometric_ellipsoid) as elevation:
ground = elevation.mean_ground(session.bounds)
tracks = make_tracks(observations, projection, elevation, ground, options.max_gap)
data = map_data(session.bounds, cache, local_map, refresh=refresh_map)
try:
water = water_areas(data)
except WaterMaskError as exc:
LOGGER.warning(
"Land elevation unavailable: %s. Reported flight altitudes are unaffected.", exc
)
water = None
terrain_source = "disabled" if no_terrain else "local terrain"
if terrain is None and not no_terrain:
terrain = terrain_data(session.bounds, cache / "terrain", refresh=refresh_terrain)
terrain_reference = "nap" if terrain else "unknown"
terrain_source = "PDOK AHN DTM dtm_05m; NAP; CC0"
if terrain is not None and terrain.resolve() == output.resolve():
raise ValueError("SVG output must not overwrite cached terrain data.")
with Elevation(terrain, terrain_reference) as elevation:
ground = elevation.mean_ground(session.bounds, water=water) if water is not None else None
land_coverage = elevation.land_coverage
land_samples = elevation.land_samples
valid_land_samples = elevation.valid_land_samples
water_samples = elevation.water_samples
tracks = make_tracks(observations, projection, options.max_gap)
if not tracks:
raise ValueError("No usable airborne tracks cross the map in the selected interval.")
if ground is None:
LOGGER.warning(
"Average ground elevation unavailable: supply a complete terrain raster "
"and its vertical reference."
"Average land elevation unavailable: no usable height map or land coverage. "
"Reported flight altitudes are unaffected."
)
missing = sum(track.average_height is None for track in tracks)
missing = sum(track.average_altitude is None for track in tracks)
if missing:
LOGGER.warning(
"%d track height(s) unavailable: compatible geometric altitude, terrain "
"and vertical-reference conversion are required.",
"%d track altitude(s) unavailable: reported barometric altitude is missing "
"on a contributing segment; no geometric-altitude fallback is used.",
missing,
)
data = map_data(session.bounds, cache, local_map, refresh=refresh_map)
row_height = options.font_height * 1.8
legend_height = (len(tracks) + 4) * row_height
terrain_attribution = terrain_source.startswith("PDOK")
legend_height = (len(tracks) + 5 + int(terrain_attribution)) * row_height
available_width = options.width - 2 * (options.margin + options.extension)
available_height = options.height - 2 * (options.margin + options.extension) - legend_height
if available_height < 25:
@@ -205,11 +231,15 @@ def render(
ET.SubElement(root, tag("desc")).text = (
f"Session {session.id}; bounds W,S,E,N={session.bounds.values}; "
f"{utc(start)} to {utc(end)} UTC. Local equirectangular projection. "
"Height is time-weighted aircraft altitude within the map minus the "
"area-sampled mean ground elevation, not terrain clearance. "
"Flight altitude is the time-weighted mean reported barometric pressure altitude "
"within the map, in metres, without terrain subtraction or vertical corrections. "
"It is not height above ground or terrain clearance. "
"Mean land elevation is a separate summary excluding mapped OSM water. "
f"Terrain source: {terrain_source}; "
f"Terrain reference: {terrain_reference}; "
f"valid land samples: {valid_land_samples}/{land_samples}; "
f"land coverage: {100 * land_coverage:.1f}%; excluded water samples: {water_samples}; "
f"mean ground elevation in metres: {ground}; "
f"geometric ellipsoid interpretation confirmed: {confirm_geometric_ellipsoid}. "
"Map data: OpenStreetMap contributors, ODbL, https://www.openstreetmap.org/copyright. "
"Flight data: The OpenSky Network, https://opensky-network.org. "
"Schaefer et al., Bringing Up OpenSky: A Large-scale ADS-B Sensor Network "
@@ -245,9 +275,10 @@ def render(
"data-callsign": track.observations[0].callsign or "unavailable",
"data-start-utc": utc(track.start),
"data-end-utc": utc(track.end),
"data-average-height-metres": (
f"{track.average_height:.3f}"
if track.average_height is not None
"data-altitude-source": "barometric",
"data-average-barometric-altitude-metres": (
f"{track.average_altitude:.3f}"
if track.average_altitude is not None
else "unavailable"
),
},
@@ -282,15 +313,18 @@ def render(
legend_y = top + map_height + options.extension + row_height
ground_label = "N/A" if ground is None else f"{ground:.1f} M"
legend = [
"FLIGHT TIMES UTC - HEIGHT ABOVE MAP AVERAGE GROUND, NOT TERRAIN CLEARANCE",
f"MAP AVERAGE GROUND ELEVATION: {ground_label} - N/A MEANS UNAVAILABLE",
"FLIGHT TIMES UTC - REPORTED BAROMETRIC ALTITUDE, NOT HEIGHT ABOVE GROUND",
f"MAP AVERAGE LAND ELEVATION: {ground_label} - N/A MEANS UNAVAILABLE",
f"LAND ELEVATION COVERAGE: {100 * land_coverage:.1f} PERCENT - MAPPED WATER EXCLUDED",
]
for index, track in enumerate(tracks, 1):
first = track.observations[0]
height_label = "N/A" if track.average_height is None else f"{track.average_height:.0f} M"
altitude_label = (
"N/A" if track.average_altitude is None else f"{track.average_altitude:.0f} M"
)
legend.append(
f"{index}: {utc(track.start)} TO {utc(track.end)} "
f"{first.callsign or 'CALLSIGN N/A'} ({first.icao24.upper()}) HEIGHT {height_label}"
f"{first.callsign or 'CALLSIGN N/A'} ({first.icao24.upper()}) BARO ALT {altitude_label}"
)
legend.extend(
[
@@ -298,6 +332,8 @@ def render(
"FLIGHTS: THE OPENSKY NETWORK - OPENSKY-NETWORK.ORG",
]
)
if terrain_attribution:
legend.append("TERRAIN: AHN DTM VIA PDOK (CC0) - ELEVATIONS RELATIVE TO NAP")
for index, text in enumerate(legend):
if len(text) * options.font_height > options.width - 2 * options.margin:
raise ValueError("Legend is too wide; reduce --font-height or increase page width.")
+44 -26
View File
@@ -1,3 +1,4 @@
import logging
import math
from contextlib import ExitStack
from pathlib import Path
@@ -5,39 +6,40 @@ from typing import Any
import rasterio
from pyproj import Transformer
from shapely.geometry import Point
from shapely.geometry.base import BaseGeometry
from flight_map.geo import Bounds
from flight_map.storage import Observation
LOGGER = logging.getLogger(__name__)
class Elevation:
"""Metre-valued terrain and optional h-H geoid separation rasters."""
"""Metre-valued terrain for the separate map land-elevation summary."""
def __init__(
self,
terrain: Path | None,
reference: str = "unknown",
geoid: Path | None = None,
confirm_geometric_ellipsoid: bool = False,
) -> None:
self.terrain_path = terrain
self.reference = reference
self.geoid_path = geoid
self.confirm_geometric_ellipsoid = confirm_geometric_ellipsoid
self.stack = ExitStack()
self.ground: Any = None
self.geoid: Any = None
self.ground_transform: Transformer | None = None
self.geoid_transform: Transformer | None = None
self.land_samples = 0
self.valid_land_samples = 0
self.water_samples = 0
@property
def land_coverage(self) -> float:
return self.valid_land_samples / self.land_samples if self.land_samples else 0.0
def __enter__(self) -> "Elevation":
with ExitStack() as stack:
if self.terrain_path:
self.ground = stack.enter_context(rasterio.open(self.terrain_path))
self.ground_transform = self._transform(self.ground)
if self.geoid_path:
self.geoid = stack.enter_context(rasterio.open(self.geoid_path))
self.geoid_transform = self._transform(self.geoid)
self.stack = stack.pop_all()
return self
@@ -64,7 +66,10 @@ class Elevation:
result = float(value) * dataset.scales[0] + dataset.offsets[0]
return result if math.isfinite(result) else None
def mean_ground(self, bounds: Bounds, samples: int = 64) -> float | None:
def mean_ground(
self, bounds: Bounds, samples: int = 64, *, water: BaseGeometry | None = None
) -> float | None:
self.land_samples = self.valid_land_samples = self.water_samples = 0
if not self.ground or self.ground_transform is None or self.reference == "unknown":
return None
if samples < 2:
@@ -81,22 +86,35 @@ class Elevation:
)
for col in range(samples):
lon = bounds.west + (bounds.east - bounds.west) * (col + 0.5) / samples
if water is not None and water.covers(Point(lon, lat)):
self.water_samples += 1
continue
self.land_samples += 1
value = self.sample(self.ground, self.ground_transform, lon, lat)
if value is None:
return None
continue
self.valid_land_samples += 1
values.append(value)
return sum(values) / len(values)
def aircraft_altitude(self, observation: Observation) -> float | None:
if not self.confirm_geometric_ellipsoid or observation.geo_altitude is None:
if not values:
LOGGER.warning(
"Average land elevation unavailable: no valid land samples "
"(%d mapped water samples excluded).",
self.water_samples,
)
return None
if self.reference == "wgs84-ellipsoid":
return observation.geo_altitude
if self.reference != "orthometric" or not self.geoid or self.geoid_transform is None:
return None
separation = self.sample(
self.geoid, self.geoid_transform, observation.longitude, observation.latitude
if self.valid_land_samples < self.land_samples:
LOGGER.warning(
"Incomplete height map: %.1f%% of sampled land has elevation data "
"(%d/%d); averaging valid land only. Missing land is not treated as water.",
100 * self.land_coverage,
self.valid_land_samples,
self.land_samples,
)
LOGGER.info(
"Land elevation coverage: %.1f%% (%d/%d); %d water samples excluded.",
100 * self.land_coverage,
self.valid_land_samples,
self.land_samples,
self.water_samples,
)
if separation is None:
return None
return observation.geo_altitude - separation
return sum(values) / len(values)
+6 -9
View File
@@ -5,14 +5,13 @@ from itertools import groupby, pairwise
from flight_map.curves import Cubic, interpolate
from flight_map.geo import Projection
from flight_map.storage import Observation
from flight_map.terrain import Elevation
@dataclass
class Track:
observations: list[Observation]
curves: list[Cubic]
average_height: float | None
average_altitude: float | None
start: float
end: float
@@ -48,8 +47,6 @@ def split_tracks(observations: list[Observation], max_gap: float) -> list[list[O
def make_tracks(
observations: list[Observation],
projection: Projection,
elevation: Elevation,
ground: float | None,
max_gap: float,
) -> list[Track]:
result = []
@@ -60,18 +57,18 @@ def make_tracks(
continue
total_time = 0.0
altitude_integral = 0.0
missing_height = ground is None
missing_altitude = False
start, end = math.inf, -math.inf
for index, (first, last) in enumerate(pairwise(items)):
duration = last.timestamp - first.timestamp
h0, h1 = elevation.aircraft_altitude(first), elevation.aircraft_altitude(last)
h0, h1 = first.baro_altitude, last.baro_altitude
for f0, f1 in curves[index].inside_intervals(projection.rectangle.bounds):
inside_duration = duration * (f1 - f0)
total_time += inside_duration
start = min(start, first.timestamp + f0 * duration)
end = max(end, first.timestamp + f1 * duration)
if h0 is None or h1 is None:
missing_height = True
missing_altitude = True
else:
entry = h0 + (h1 - h0) * f0
exit_height = h0 + (h1 - h0) * f1
@@ -79,7 +76,7 @@ def make_tracks(
if total_time == 0:
continue
average = None
if not missing_height and ground is not None:
average = altitude_integral / total_time - ground
if not missing_altitude:
average = altitude_integral / total_time
result.append(Track(items, curves, average, start, end))
return sorted(result, key=lambda track: (track.start, track.observations[0].icao24))