feat: implement StaySense MVP backend, frontend, imports, and deployment docs
This commit is contained in:
commit
902988276c
24 changed files with 2536 additions and 0 deletions
122
backend/db.py
Normal file
122
backend/db.py
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
BASE_DIR = Path(__file__).resolve().parent.parent
|
||||
DB_PATH = BASE_DIR / "data" / "staysense.db"
|
||||
|
||||
|
||||
def get_conn() -> sqlite3.Connection:
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
return conn
|
||||
|
||||
|
||||
def init_db() -> None:
|
||||
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
with get_conn() as conn:
|
||||
conn.executescript(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS spot (
|
||||
id TEXT PRIMARY KEY,
|
||||
lat REAL NOT NULL,
|
||||
lon REAL NOT NULL,
|
||||
osm_area_type TEXT NOT NULL CHECK (osm_area_type IN ('residential', 'industrial', 'commercial', 'parking', 'nature')),
|
||||
road_type TEXT NOT NULL CHECK (road_type IN ('residential', 'primary', 'secondary', 'service', 'unknown')),
|
||||
distance_police_m INTEGER NOT NULL,
|
||||
distance_fire_m INTEGER NOT NULL,
|
||||
distance_hospital_m INTEGER NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS local_event (
|
||||
id TEXT PRIMARY KEY,
|
||||
spot_id TEXT NOT NULL,
|
||||
event_type TEXT NOT NULL CHECK (event_type IN ('market', 'waste', 'event', 'construction')),
|
||||
start_datetime TEXT NOT NULL,
|
||||
end_datetime TEXT NOT NULL,
|
||||
risk_modifier INTEGER NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
FOREIGN KEY (spot_id) REFERENCES spot(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS community_signal (
|
||||
id TEXT PRIMARY KEY,
|
||||
spot_id TEXT NOT NULL,
|
||||
signal_type TEXT NOT NULL CHECK (signal_type IN ('calm', 'noise', 'knock', 'police')),
|
||||
hashed_device TEXT NOT NULL,
|
||||
timestamp TEXT NOT NULL,
|
||||
day_bucket TEXT NOT NULL,
|
||||
FOREIGN KEY (spot_id) REFERENCES spot(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS osm_poi (
|
||||
id TEXT PRIMARY KEY,
|
||||
poi_type TEXT NOT NULL CHECK (poi_type IN ('police', 'fire', 'hospital')),
|
||||
lat REAL NOT NULL,
|
||||
lon REAL NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
imported_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS osm_zone (
|
||||
id TEXT PRIMARY KEY,
|
||||
zone_type TEXT NOT NULL CHECK (zone_type IN ('residential', 'industrial', 'commercial', 'parking', 'nature')),
|
||||
lat REAL NOT NULL,
|
||||
lon REAL NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
imported_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS osm_road (
|
||||
id TEXT PRIMARY KEY,
|
||||
road_type TEXT NOT NULL CHECK (road_type IN ('residential', 'primary', 'secondary', 'service')),
|
||||
lat REAL NOT NULL,
|
||||
lon REAL NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
imported_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS open_data_event (
|
||||
id TEXT PRIMARY KEY,
|
||||
event_type TEXT NOT NULL CHECK (event_type IN ('market', 'waste', 'event', 'construction')),
|
||||
lat REAL NOT NULL,
|
||||
lon REAL NOT NULL,
|
||||
start_datetime TEXT NOT NULL,
|
||||
end_datetime TEXT NOT NULL,
|
||||
risk_modifier INTEGER NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
imported_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS data_source_state (
|
||||
source_name TEXT PRIMARY KEY,
|
||||
imported_at TEXT NOT NULL,
|
||||
record_count INTEGER NOT NULL,
|
||||
notes TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_signal_spot_timestamp
|
||||
ON community_signal (spot_id, timestamp);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS ux_signal_spot_device_day
|
||||
ON community_signal (spot_id, hashed_device, day_bucket);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_local_event_spot_window
|
||||
ON local_event (spot_id, start_datetime, end_datetime);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_osm_poi_type
|
||||
ON osm_poi (poi_type);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_osm_zone_type
|
||||
ON osm_zone (zone_type);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_open_data_event_window
|
||||
ON open_data_event (start_datetime, end_datetime);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_open_data_event_source
|
||||
ON open_data_event (source);
|
||||
"""
|
||||
)
|
||||
52
backend/import_open_data_events.py
Normal file
52
backend/import_open_data_events.py
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
import argparse
|
||||
import csv
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from open_data_connector import import_event_rows
|
||||
|
||||
|
||||
def import_events(csv_path: Path) -> dict:
|
||||
rows = []
|
||||
with csv_path.open("r", encoding="utf-8") as handle:
|
||||
reader = csv.DictReader(handle)
|
||||
required = {"lat", "lon", "event_type", "start_datetime", "end_datetime", "risk_modifier"}
|
||||
if not required.issubset(set(reader.fieldnames or [])):
|
||||
raise ValueError(f"CSV missing required columns: {sorted(required)}")
|
||||
|
||||
for line in reader:
|
||||
rows.append(
|
||||
{
|
||||
"id": None,
|
||||
"event_type": (line.get("event_type") or "").strip(),
|
||||
"lat": (line.get("lat") or "").strip(),
|
||||
"lon": (line.get("lon") or "").strip(),
|
||||
"start_datetime": (line.get("start_datetime") or "").strip(),
|
||||
"end_datetime": (line.get("end_datetime") or "").strip(),
|
||||
"risk_modifier": (line.get("risk_modifier") or "0").strip(),
|
||||
}
|
||||
)
|
||||
|
||||
# Legacy CSV import keeps source-name bound to filename.
|
||||
source_name = f"open_data_events:{csv_path.name}"
|
||||
normalized = []
|
||||
for idx, row in enumerate(rows):
|
||||
row["id"] = f"{source_name}:{idx}"
|
||||
normalized.append(row)
|
||||
|
||||
result = import_event_rows(normalized, source_name=source_name, notes=str(csv_path))
|
||||
result["source"] = str(csv_path)
|
||||
return result
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Import Open Data events CSV for StaySense")
|
||||
parser.add_argument("--file", required=True, help="CSV with lat,lon,event_type,start_datetime,end_datetime,risk_modifier,source")
|
||||
args = parser.parse_args()
|
||||
|
||||
result = import_events(Path(args.file))
|
||||
print(json.dumps(result, ensure_ascii=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
209
backend/import_osm_overpass.py
Normal file
209
backend/import_osm_overpass.py
Normal file
|
|
@ -0,0 +1,209 @@
|
|||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import uuid
|
||||
from urllib import parse, request
|
||||
|
||||
from db import get_conn, init_db
|
||||
|
||||
OVERPASS_URL = "https://overpass-api.de/api/interpreter"
|
||||
DEFAULT_BBOX = (51.16, 6.79, 51.38, 7.15) # Kreis Mettmann approx: s,w,n,e
|
||||
|
||||
|
||||
def now_iso() -> str:
|
||||
return dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def build_query(south: float, west: float, north: float, east: float) -> str:
|
||||
bbox = f"({south},{west},{north},{east})"
|
||||
return f"""
|
||||
[out:json][timeout:120];
|
||||
(
|
||||
node[amenity=police]{bbox};
|
||||
way[amenity=police]{bbox};
|
||||
relation[amenity=police]{bbox};
|
||||
|
||||
node[amenity=fire_station]{bbox};
|
||||
way[amenity=fire_station]{bbox};
|
||||
relation[amenity=fire_station]{bbox};
|
||||
|
||||
node[amenity=hospital]{bbox};
|
||||
way[amenity=hospital]{bbox};
|
||||
relation[amenity=hospital]{bbox};
|
||||
|
||||
way[landuse=residential]{bbox};
|
||||
relation[landuse=residential]{bbox};
|
||||
|
||||
way[landuse=industrial]{bbox};
|
||||
relation[landuse=industrial]{bbox};
|
||||
|
||||
way[landuse=commercial]{bbox};
|
||||
relation[landuse=commercial]{bbox};
|
||||
|
||||
node[amenity=parking]{bbox};
|
||||
way[amenity=parking]{bbox};
|
||||
relation[amenity=parking]{bbox};
|
||||
|
||||
way[natural=wood]{bbox};
|
||||
relation[natural=wood]{bbox};
|
||||
way[leisure=nature_reserve]{bbox};
|
||||
relation[leisure=nature_reserve]{bbox};
|
||||
|
||||
way[highway=primary]{bbox};
|
||||
way[highway=secondary]{bbox};
|
||||
way[highway=residential]{bbox};
|
||||
way[highway=service]{bbox};
|
||||
);
|
||||
out center;
|
||||
"""
|
||||
|
||||
|
||||
def element_coords(element: dict) -> tuple[float, float] | None:
|
||||
if "lat" in element and "lon" in element:
|
||||
return float(element["lat"]), float(element["lon"])
|
||||
center = element.get("center")
|
||||
if center and "lat" in center and "lon" in center:
|
||||
return float(center["lat"]), float(center["lon"])
|
||||
return None
|
||||
|
||||
|
||||
def map_poi(tags: dict) -> str | None:
|
||||
amenity = tags.get("amenity")
|
||||
if amenity == "police":
|
||||
return "police"
|
||||
if amenity == "fire_station":
|
||||
return "fire"
|
||||
if amenity == "hospital":
|
||||
return "hospital"
|
||||
return None
|
||||
|
||||
|
||||
def map_zone(tags: dict) -> str | None:
|
||||
landuse = tags.get("landuse")
|
||||
amenity = tags.get("amenity")
|
||||
natural = tags.get("natural")
|
||||
leisure = tags.get("leisure")
|
||||
|
||||
if landuse == "residential":
|
||||
return "residential"
|
||||
if landuse == "industrial":
|
||||
return "industrial"
|
||||
if landuse == "commercial":
|
||||
return "commercial"
|
||||
if amenity == "parking":
|
||||
return "parking"
|
||||
if natural in {"wood", "scrub", "heath"} or leisure == "nature_reserve":
|
||||
return "nature"
|
||||
return None
|
||||
|
||||
|
||||
def map_road(tags: dict) -> str | None:
|
||||
highway = tags.get("highway")
|
||||
if highway in {"primary", "secondary", "residential", "service"}:
|
||||
return highway
|
||||
return None
|
||||
|
||||
|
||||
def fetch_overpass(query: str) -> dict:
|
||||
payload = parse.urlencode({"data": query}).encode("utf-8")
|
||||
req = request.Request(OVERPASS_URL, data=payload, method="POST")
|
||||
with request.urlopen(req, timeout=180) as resp:
|
||||
raw = resp.read()
|
||||
return json.loads(raw.decode("utf-8"))
|
||||
|
||||
|
||||
def import_osm(south: float, west: float, north: float, east: float) -> dict:
|
||||
init_db()
|
||||
data = fetch_overpass(build_query(south, west, north, east))
|
||||
elements = data.get("elements", [])
|
||||
|
||||
imported_at = now_iso()
|
||||
source = "osm_overpass"
|
||||
|
||||
poi_rows = []
|
||||
zone_rows = []
|
||||
road_rows = []
|
||||
|
||||
for element in elements:
|
||||
tags = element.get("tags", {})
|
||||
coords = element_coords(element)
|
||||
if not coords:
|
||||
continue
|
||||
|
||||
lat, lon = coords
|
||||
base_id = f"{element.get('type', 'x')}:{element.get('id', uuid.uuid4())}"
|
||||
|
||||
poi_type = map_poi(tags)
|
||||
if poi_type:
|
||||
poi_rows.append((f"poi:{base_id}", poi_type, lat, lon, source, imported_at))
|
||||
|
||||
zone_type = map_zone(tags)
|
||||
if zone_type:
|
||||
zone_rows.append((f"zone:{base_id}", zone_type, lat, lon, source, imported_at))
|
||||
|
||||
road_type = map_road(tags)
|
||||
if road_type:
|
||||
road_rows.append((f"road:{base_id}", road_type, lat, lon, source, imported_at))
|
||||
|
||||
with get_conn() as conn:
|
||||
conn.execute("DELETE FROM osm_poi WHERE source = ?", (source,))
|
||||
conn.execute("DELETE FROM osm_zone WHERE source = ?", (source,))
|
||||
conn.execute("DELETE FROM osm_road WHERE source = ?", (source,))
|
||||
|
||||
conn.executemany(
|
||||
"""
|
||||
INSERT INTO osm_poi (id, poi_type, lat, lon, source, imported_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
poi_rows,
|
||||
)
|
||||
conn.executemany(
|
||||
"""
|
||||
INSERT INTO osm_zone (id, zone_type, lat, lon, source, imported_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
zone_rows,
|
||||
)
|
||||
conn.executemany(
|
||||
"""
|
||||
INSERT INTO osm_road (id, road_type, lat, lon, source, imported_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
road_rows,
|
||||
)
|
||||
|
||||
total = len(poi_rows) + len(zone_rows) + len(road_rows)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO data_source_state (source_name, imported_at, record_count, notes)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(source_name) DO UPDATE SET
|
||||
imported_at = excluded.imported_at,
|
||||
record_count = excluded.record_count,
|
||||
notes = excluded.notes
|
||||
""",
|
||||
("osm_overpass", imported_at, total, f"bbox={south},{west},{north},{east}"),
|
||||
)
|
||||
|
||||
return {
|
||||
"elements": len(elements),
|
||||
"pois": len(poi_rows),
|
||||
"zones": len(zone_rows),
|
||||
"roads": len(road_rows),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Import OSM data for StaySense from Overpass")
|
||||
parser.add_argument("--south", type=float, default=DEFAULT_BBOX[0])
|
||||
parser.add_argument("--west", type=float, default=DEFAULT_BBOX[1])
|
||||
parser.add_argument("--north", type=float, default=DEFAULT_BBOX[2])
|
||||
parser.add_argument("--east", type=float, default=DEFAULT_BBOX[3])
|
||||
args = parser.parse_args()
|
||||
|
||||
result = import_osm(args.south, args.west, args.north, args.east)
|
||||
print(json.dumps(result, ensure_ascii=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
317
backend/open_data_connector.py
Normal file
317
backend/open_data_connector.py
Normal file
|
|
@ -0,0 +1,317 @@
|
|||
import csv
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from urllib import request
|
||||
|
||||
from db import get_conn, init_db
|
||||
|
||||
VALID_TYPES = {"market", "waste", "event", "construction"}
|
||||
DE_BOUNDS = {"lat_min": 47.0, "lat_max": 55.5, "lon_min": 5.0, "lon_max": 16.0}
|
||||
|
||||
|
||||
def now_iso() -> str:
|
||||
return dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def normalize_iso(value: str) -> str:
|
||||
parsed = dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=dt.timezone.utc)
|
||||
return parsed.astimezone(dt.timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _stable_id(source_name: str, external_id: str | None, payload: dict) -> str:
|
||||
if external_id:
|
||||
raw = f"{source_name}:{external_id}"
|
||||
else:
|
||||
raw = (
|
||||
f"{source_name}:{payload['event_type']}:{payload['lat']:.6f}:{payload['lon']:.6f}:"
|
||||
f"{payload['start_datetime']}:{payload['end_datetime']}:{payload['risk_modifier']}"
|
||||
)
|
||||
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _map_event_type(raw_value: str, event_type_map: dict[str, str], default_value: str | None) -> str | None:
|
||||
raw = (raw_value or "").strip().lower()
|
||||
if raw and raw in event_type_map:
|
||||
raw = event_type_map[raw]
|
||||
if not raw and default_value:
|
||||
raw = default_value
|
||||
if raw in VALID_TYPES:
|
||||
return raw
|
||||
return None
|
||||
|
||||
|
||||
def _extract_json_records(payload: dict | list, json_path: str | None) -> list[dict]:
|
||||
current = payload
|
||||
if json_path:
|
||||
for part in json_path.split("."):
|
||||
if isinstance(current, dict):
|
||||
current = current.get(part)
|
||||
else:
|
||||
current = None
|
||||
if current is None:
|
||||
return []
|
||||
if isinstance(current, list):
|
||||
return [item for item in current if isinstance(item, dict)]
|
||||
return []
|
||||
|
||||
|
||||
def _read_text(location: str, config_dir: Path) -> str:
|
||||
if location.startswith("http://") or location.startswith("https://"):
|
||||
with request.urlopen(location, timeout=60) as resp:
|
||||
return resp.read().decode("utf-8", errors="replace")
|
||||
|
||||
file_path = Path(location)
|
||||
if not file_path.is_absolute():
|
||||
file_path = (config_dir / location).resolve()
|
||||
return file_path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def import_event_rows(rows: list[dict], source_name: str, notes: str) -> dict:
|
||||
init_db()
|
||||
imported_at = now_iso()
|
||||
|
||||
db_rows = []
|
||||
for item in rows:
|
||||
event_type = item.get("event_type")
|
||||
if event_type not in VALID_TYPES:
|
||||
continue
|
||||
|
||||
db_rows.append(
|
||||
(
|
||||
item["id"],
|
||||
event_type,
|
||||
float(item["lat"]),
|
||||
float(item["lon"]),
|
||||
normalize_iso(item["start_datetime"]),
|
||||
normalize_iso(item["end_datetime"]),
|
||||
int(item["risk_modifier"]),
|
||||
source_name,
|
||||
imported_at,
|
||||
)
|
||||
)
|
||||
|
||||
with get_conn() as conn:
|
||||
conn.execute("DELETE FROM open_data_event WHERE source = ?", (source_name,))
|
||||
conn.executemany(
|
||||
"""
|
||||
INSERT INTO open_data_event (
|
||||
id, event_type, lat, lon, start_datetime, end_datetime,
|
||||
risk_modifier, source, imported_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
db_rows,
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO data_source_state (source_name, imported_at, record_count, notes)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(source_name) DO UPDATE SET
|
||||
imported_at = excluded.imported_at,
|
||||
record_count = excluded.record_count,
|
||||
notes = excluded.notes
|
||||
""",
|
||||
(source_name, imported_at, len(db_rows), notes),
|
||||
)
|
||||
|
||||
return {"rows": len(db_rows), "source_name": source_name}
|
||||
|
||||
|
||||
def import_from_source(source_cfg: dict, config_dir: Path) -> dict:
|
||||
source_name = source_cfg.get("source_name") or source_cfg.get("id") or "open_data_source"
|
||||
location = source_cfg.get("url") or source_cfg.get("file")
|
||||
if not location:
|
||||
raise ValueError(f"source '{source_name}' missing url/file")
|
||||
|
||||
source_format = (source_cfg.get("format") or "csv").lower()
|
||||
field_map = source_cfg.get("field_map") or {}
|
||||
defaults = source_cfg.get("default_values") or {}
|
||||
event_type_map = {str(k).lower(): str(v).lower() for k, v in (source_cfg.get("event_type_map") or {}).items()}
|
||||
json_path = source_cfg.get("json_path")
|
||||
|
||||
text = _read_text(location, config_dir)
|
||||
|
||||
records: list[dict]
|
||||
if source_format == "csv":
|
||||
reader = csv.DictReader(text.splitlines())
|
||||
records = [dict(row) for row in reader]
|
||||
elif source_format == "json":
|
||||
payload = json.loads(text)
|
||||
records = _extract_json_records(payload, json_path)
|
||||
else:
|
||||
raise ValueError(f"unsupported format: {source_format}")
|
||||
|
||||
rows = []
|
||||
stats = {
|
||||
"input_records": len(records),
|
||||
"accepted": 0,
|
||||
"rejected_invalid_event_type": 0,
|
||||
"rejected_parse_error": 0,
|
||||
"rejected_missing_datetime": 0,
|
||||
"rejected_out_of_bounds": 0,
|
||||
"rejected_invalid_window": 0,
|
||||
}
|
||||
for record in records:
|
||||
lat_key = field_map.get("lat", "lat")
|
||||
lon_key = field_map.get("lon", "lon")
|
||||
start_key = field_map.get("start_datetime", "start_datetime")
|
||||
end_key = field_map.get("end_datetime", "end_datetime")
|
||||
risk_key = field_map.get("risk_modifier", "risk_modifier")
|
||||
event_type_key = field_map.get("event_type", "event_type")
|
||||
external_id_key = field_map.get("external_id")
|
||||
|
||||
raw_event_type = str(record.get(event_type_key, defaults.get("event_type", "")))
|
||||
event_type = _map_event_type(raw_event_type, event_type_map, defaults.get("event_type"))
|
||||
if not event_type:
|
||||
stats["rejected_invalid_event_type"] += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
lat = float(str(record.get(lat_key, defaults.get("lat", ""))).strip())
|
||||
lon = float(str(record.get(lon_key, defaults.get("lon", ""))).strip())
|
||||
start_dt = str(record.get(start_key, defaults.get("start_datetime", "")).strip())
|
||||
end_dt = str(record.get(end_key, defaults.get("end_datetime", "")).strip())
|
||||
risk_modifier = int(str(record.get(risk_key, defaults.get("risk_modifier", "0"))).strip())
|
||||
except Exception:
|
||||
stats["rejected_parse_error"] += 1
|
||||
continue
|
||||
|
||||
if not start_dt or not end_dt:
|
||||
stats["rejected_missing_datetime"] += 1
|
||||
continue
|
||||
|
||||
if not (
|
||||
DE_BOUNDS["lat_min"] <= lat <= DE_BOUNDS["lat_max"]
|
||||
and DE_BOUNDS["lon_min"] <= lon <= DE_BOUNDS["lon_max"]
|
||||
):
|
||||
stats["rejected_out_of_bounds"] += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
start_iso = normalize_iso(start_dt)
|
||||
end_iso = normalize_iso(end_dt)
|
||||
except Exception:
|
||||
stats["rejected_parse_error"] += 1
|
||||
continue
|
||||
|
||||
if end_iso <= start_iso:
|
||||
stats["rejected_invalid_window"] += 1
|
||||
continue
|
||||
|
||||
external_id = None
|
||||
if external_id_key:
|
||||
value = record.get(external_id_key)
|
||||
external_id = str(value).strip() if value is not None else None
|
||||
|
||||
payload = {
|
||||
"event_type": event_type,
|
||||
"lat": lat,
|
||||
"lon": lon,
|
||||
"start_datetime": start_iso,
|
||||
"end_datetime": end_iso,
|
||||
"risk_modifier": risk_modifier,
|
||||
}
|
||||
payload["id"] = _stable_id(source_name, external_id, payload)
|
||||
rows.append(payload)
|
||||
stats["accepted"] += 1
|
||||
|
||||
note = f"connector={source_cfg.get('id', source_name)} location={location} format={source_format}"
|
||||
result = import_event_rows(rows, source_name=source_name, notes=note)
|
||||
result["connector_id"] = source_cfg.get("id", source_name)
|
||||
result["stats"] = stats
|
||||
return result
|
||||
|
||||
|
||||
def prune_sources_not_in_config(config_source_names: set[str], protected_sources: set[str] | None = None) -> dict:
|
||||
protected = protected_sources or {"osm_overpass"}
|
||||
with get_conn() as conn:
|
||||
rows = conn.execute("SELECT DISTINCT source FROM open_data_event").fetchall()
|
||||
existing_sources = {row["source"] for row in rows}
|
||||
state_rows = conn.execute("SELECT source_name FROM data_source_state").fetchall()
|
||||
existing_state_sources = {row["source_name"] for row in state_rows}
|
||||
|
||||
to_prune = sorted((existing_sources - config_source_names) - protected)
|
||||
state_to_mark = sorted((existing_state_sources - config_source_names) - protected)
|
||||
pruned_rows = 0
|
||||
ts = now_iso()
|
||||
for source_name in to_prune:
|
||||
count = conn.execute("SELECT COUNT(*) AS c FROM open_data_event WHERE source = ?", (source_name,)).fetchone()["c"]
|
||||
pruned_rows += int(count)
|
||||
conn.execute("DELETE FROM open_data_event WHERE source = ?", (source_name,))
|
||||
for source_name in state_to_mark:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO data_source_state (source_name, imported_at, record_count, notes)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(source_name) DO UPDATE SET
|
||||
imported_at = excluded.imported_at,
|
||||
record_count = excluded.record_count,
|
||||
notes = excluded.notes
|
||||
""",
|
||||
(source_name, ts, 0, "pruned_not_in_config"),
|
||||
)
|
||||
|
||||
return {"sources": sorted(set(to_prune + state_to_mark)), "rows": pruned_rows}
|
||||
|
||||
|
||||
def import_from_config(config_path: Path, prune_legacy: bool = False) -> dict:
|
||||
cfg = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
sources = cfg.get("sources")
|
||||
if not isinstance(sources, list):
|
||||
raise ValueError("config requires 'sources' list")
|
||||
|
||||
imported = []
|
||||
skipped = []
|
||||
disabled_source_names = []
|
||||
all_source_names = set()
|
||||
for source in sources:
|
||||
if not isinstance(source, dict):
|
||||
continue
|
||||
source_name = source.get("source_name") or source.get("id") or "open_data_source"
|
||||
all_source_names.add(source_name)
|
||||
if not source.get("enabled", False):
|
||||
skipped.append(source.get("id", "unknown"))
|
||||
disabled_source_names.append(source_name)
|
||||
continue
|
||||
try:
|
||||
imported.append(import_from_source(source, config_path.parent))
|
||||
except Exception as exc:
|
||||
imported.append(
|
||||
{
|
||||
"connector_id": source.get("id", source_name),
|
||||
"source_name": source_name,
|
||||
"rows": 0,
|
||||
"error": str(exc),
|
||||
}
|
||||
)
|
||||
|
||||
if disabled_source_names:
|
||||
ts = now_iso()
|
||||
with get_conn() as conn:
|
||||
for source_name in disabled_source_names:
|
||||
conn.execute("DELETE FROM open_data_event WHERE source = ?", (source_name,))
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO data_source_state (source_name, imported_at, record_count, notes)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(source_name) DO UPDATE SET
|
||||
imported_at = excluded.imported_at,
|
||||
record_count = excluded.record_count,
|
||||
notes = excluded.notes
|
||||
""",
|
||||
(source_name, ts, 0, "disabled in config"),
|
||||
)
|
||||
|
||||
pruned = {"sources": [], "rows": 0}
|
||||
if prune_legacy:
|
||||
pruned = prune_sources_not_in_config(all_source_names)
|
||||
|
||||
return {
|
||||
"imported": imported,
|
||||
"skipped": skipped,
|
||||
"pruned": pruned,
|
||||
"count": len(imported),
|
||||
}
|
||||
43
backend/run_import_jobs.py
Normal file
43
backend/run_import_jobs.py
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
import argparse
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from import_osm_overpass import DEFAULT_BBOX, import_osm
|
||||
from open_data_connector import import_from_config
|
||||
|
||||
|
||||
def run_once(config_path: Path, with_osm: bool, prune_legacy: bool, south: float, west: float, north: float, east: float) -> dict:
|
||||
result = {"open_data": import_from_config(config_path, prune_legacy=prune_legacy)}
|
||||
if with_osm:
|
||||
result["osm"] = import_osm(south, west, north, east)
|
||||
return result
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Run StaySense data import jobs")
|
||||
parser.add_argument("--config", default="../docs/open_data_sources.json", help="Path to open data source config JSON")
|
||||
parser.add_argument("--with-osm", action="store_true", help="Also run OSM overpass import")
|
||||
parser.add_argument("--prune-legacy", action="store_true", help="Delete open_data_event sources not present in config")
|
||||
parser.add_argument("--daemon", action="store_true", help="Run continuously")
|
||||
parser.add_argument("--interval-seconds", type=int, default=21600, help="Loop interval for daemon mode")
|
||||
parser.add_argument("--south", type=float, default=DEFAULT_BBOX[0])
|
||||
parser.add_argument("--west", type=float, default=DEFAULT_BBOX[1])
|
||||
parser.add_argument("--north", type=float, default=DEFAULT_BBOX[2])
|
||||
parser.add_argument("--east", type=float, default=DEFAULT_BBOX[3])
|
||||
args = parser.parse_args()
|
||||
|
||||
config_path = Path(args.config).resolve()
|
||||
|
||||
if not args.daemon:
|
||||
print(json.dumps(run_once(config_path, args.with_osm, args.prune_legacy, args.south, args.west, args.north, args.east), ensure_ascii=True))
|
||||
return
|
||||
|
||||
while True:
|
||||
result = run_once(config_path, args.with_osm, args.prune_legacy, args.south, args.west, args.north, args.east)
|
||||
print(json.dumps(result, ensure_ascii=True))
|
||||
time.sleep(max(60, args.interval_seconds))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
153
backend/score_engine.py
Normal file
153
backend/score_engine.py
Normal file
|
|
@ -0,0 +1,153 @@
|
|||
import datetime as dt
|
||||
import math
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
|
||||
NRW_HOLIDAYS_2026 = {
|
||||
dt.date(2026, 1, 1),
|
||||
dt.date(2026, 4, 3),
|
||||
dt.date(2026, 4, 6),
|
||||
dt.date(2026, 5, 1),
|
||||
dt.date(2026, 5, 14),
|
||||
dt.date(2026, 5, 25),
|
||||
dt.date(2026, 6, 4),
|
||||
dt.date(2026, 10, 3),
|
||||
dt.date(2026, 11, 1),
|
||||
dt.date(2026, 12, 25),
|
||||
dt.date(2026, 12, 26),
|
||||
}
|
||||
|
||||
POLICE_POINTS = [
|
||||
(51.2507, 6.9751), # Mettmann
|
||||
(51.2965, 6.8494), # Ratingen
|
||||
(51.3398, 7.0438), # Velbert
|
||||
]
|
||||
|
||||
FIRE_POINTS = [
|
||||
(51.2518, 6.9800),
|
||||
(51.2937, 6.8568),
|
||||
(51.3314, 7.0540),
|
||||
]
|
||||
|
||||
HOSPITAL_POINTS = [
|
||||
(51.2556, 6.9723),
|
||||
(51.2891, 6.8457),
|
||||
(51.3321, 7.0403),
|
||||
]
|
||||
|
||||
INDUSTRIAL_ZONES = [
|
||||
(51.2348, 6.9902),
|
||||
(51.3022, 6.8340),
|
||||
]
|
||||
COMMERCIAL_ZONES = [
|
||||
(51.2512, 6.9878),
|
||||
(51.2934, 6.8541),
|
||||
]
|
||||
NATURE_ZONES = [
|
||||
(51.2715, 6.9440),
|
||||
(51.3188, 7.0274),
|
||||
]
|
||||
PARKING_ZONES = [
|
||||
(51.2499, 6.9831),
|
||||
]
|
||||
MAIN_ROAD_POINTS = [
|
||||
(51.2524, 6.9921),
|
||||
(51.2951, 6.8615),
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class Factor:
|
||||
key: str
|
||||
label: str
|
||||
points: float
|
||||
|
||||
|
||||
def haversine_m(lat1: float, lon1: float, lat2: float, lon2: float) -> float:
|
||||
r = 6371000
|
||||
p1 = math.radians(lat1)
|
||||
p2 = math.radians(lat2)
|
||||
dp = math.radians(lat2 - lat1)
|
||||
dl = math.radians(lon2 - lon1)
|
||||
a = math.sin(dp / 2) ** 2 + math.cos(p1) * math.cos(p2) * math.sin(dl / 2) ** 2
|
||||
return 2 * r * math.asin(math.sqrt(a))
|
||||
|
||||
|
||||
def nearest_distance_m(lat: float, lon: float, points: list[tuple[float, float]]) -> int:
|
||||
return int(min(haversine_m(lat, lon, p_lat, p_lon) for p_lat, p_lon in points))
|
||||
|
||||
|
||||
def classify_area(lat: float, lon: float) -> str:
|
||||
candidates = [
|
||||
("industrial", nearest_distance_m(lat, lon, INDUSTRIAL_ZONES)),
|
||||
("commercial", nearest_distance_m(lat, lon, COMMERCIAL_ZONES)),
|
||||
("nature", nearest_distance_m(lat, lon, NATURE_ZONES)),
|
||||
("parking", nearest_distance_m(lat, lon, PARKING_ZONES)),
|
||||
]
|
||||
best_type, best_dist = min(candidates, key=lambda x: x[1])
|
||||
if best_dist <= 400:
|
||||
return best_type
|
||||
return "residential"
|
||||
|
||||
|
||||
def classify_road(lat: float, lon: float) -> str:
|
||||
if nearest_distance_m(lat, lon, MAIN_ROAD_POINTS) < 250:
|
||||
return "primary"
|
||||
return "residential"
|
||||
|
||||
|
||||
def spot_id_for(lat: float, lon: float) -> str:
|
||||
key = f"{lat:.4f}:{lon:.4f}"
|
||||
return str(uuid.uuid5(uuid.NAMESPACE_DNS, f"staysense:{key}"))
|
||||
|
||||
|
||||
def score_area_modifier(area_type: str) -> Factor:
|
||||
mapping = {
|
||||
"residential": Factor("area", "Wohngebiet", -10),
|
||||
"industrial": Factor("area", "Industriegebiet", 10),
|
||||
"commercial": Factor("area", "Gewerbegebiet", 6),
|
||||
"parking": Factor("area", "Parkplatzumfeld", -5),
|
||||
"nature": Factor("area", "Naturnah", 8),
|
||||
}
|
||||
return mapping.get(area_type, Factor("area", "Umgebung", 0))
|
||||
|
||||
|
||||
def night_window_for(reference: dt.datetime) -> tuple[dt.datetime, dt.datetime]:
|
||||
ref = reference.replace(second=0, microsecond=0)
|
||||
if ref.hour >= 22:
|
||||
start = ref.replace(hour=22, minute=0)
|
||||
elif ref.hour < 6:
|
||||
start = (ref - dt.timedelta(days=1)).replace(hour=22, minute=0)
|
||||
else:
|
||||
start = ref.replace(hour=22, minute=0)
|
||||
end = start + dt.timedelta(hours=8)
|
||||
return start, end
|
||||
|
||||
|
||||
def weekend_or_holiday(night_start: dt.datetime) -> bool:
|
||||
weekday = night_start.weekday() # 0=Mon, 4=Fri
|
||||
if weekday in (4, 5):
|
||||
return True
|
||||
if night_start.date() in NRW_HOLIDAYS_2026:
|
||||
return True
|
||||
if (night_start + dt.timedelta(days=1)).date() in NRW_HOLIDAYS_2026:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def decay(age_days: float, half_life_days: float) -> float:
|
||||
if age_days <= 0:
|
||||
return 1.0
|
||||
return 0.5 ** (age_days / half_life_days)
|
||||
|
||||
|
||||
def clamp_score(value: float) -> int:
|
||||
return max(0, min(100, int(round(value))))
|
||||
|
||||
|
||||
def ampel(score: int) -> str:
|
||||
if score >= 70:
|
||||
return "green"
|
||||
if score >= 45:
|
||||
return "yellow"
|
||||
return "red"
|
||||
512
backend/server.py
Normal file
512
backend/server.py
Normal file
|
|
@ -0,0 +1,512 @@
|
|||
import datetime as dt
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
from http import HTTPStatus
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from db import get_conn, init_db
|
||||
from score_engine import (
|
||||
ampel,
|
||||
clamp_score,
|
||||
classify_area,
|
||||
classify_road,
|
||||
decay,
|
||||
haversine_m,
|
||||
nearest_distance_m,
|
||||
night_window_for,
|
||||
score_area_modifier,
|
||||
spot_id_for,
|
||||
weekend_or_holiday,
|
||||
)
|
||||
|
||||
HOST = "127.0.0.1"
|
||||
PORT = 8787
|
||||
|
||||
DEFAULT_SALT = "change-me-in-production"
|
||||
SERVER_SALT = os.environ.get("STAYSENSE_SERVER_SALT", DEFAULT_SALT)
|
||||
SIGNAL_COOLDOWN_HOURS = int(os.environ.get("STAYSENSE_SIGNAL_COOLDOWN_HOURS", "24"))
|
||||
|
||||
FALLBACK_POLICE_POINTS = [(51.2507, 6.9751), (51.2965, 6.8494), (51.3398, 7.0438)]
|
||||
FALLBACK_FIRE_POINTS = [(51.2518, 6.9800), (51.2937, 6.8568), (51.3314, 7.0540)]
|
||||
FALLBACK_HOSPITAL_POINTS = [(51.2556, 6.9723), (51.2891, 6.8457), (51.3321, 7.0403)]
|
||||
|
||||
|
||||
def utc_now() -> dt.datetime:
|
||||
return dt.datetime.now(dt.timezone.utc)
|
||||
|
||||
|
||||
def parse_iso8601(value: str | None) -> dt.datetime:
|
||||
if not value:
|
||||
return utc_now()
|
||||
parsed = dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
if parsed.tzinfo is None:
|
||||
return parsed.replace(tzinfo=dt.timezone.utc)
|
||||
return parsed.astimezone(dt.timezone.utc)
|
||||
|
||||
|
||||
def to_iso(value: dt.datetime) -> str:
|
||||
return value.astimezone(dt.timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def json_response(handler: BaseHTTPRequestHandler, status: int, payload: dict) -> None:
|
||||
raw = json.dumps(payload, ensure_ascii=True).encode("utf-8")
|
||||
handler.send_response(status)
|
||||
handler.send_header("Content-Type", "application/json; charset=utf-8")
|
||||
handler.send_header("Content-Length", str(len(raw)))
|
||||
handler.send_header("Cache-Control", "no-store")
|
||||
handler.end_headers()
|
||||
handler.wfile.write(raw)
|
||||
|
||||
|
||||
def read_json(handler: BaseHTTPRequestHandler) -> dict:
|
||||
length = int(handler.headers.get("Content-Length", "0"))
|
||||
if length <= 0:
|
||||
return {}
|
||||
body = handler.rfile.read(length)
|
||||
return json.loads(body.decode("utf-8"))
|
||||
|
||||
|
||||
def hashed_device(device_token: str) -> str:
|
||||
return hmac.new(
|
||||
key=SERVER_SALT.encode("utf-8"),
|
||||
msg=device_token.encode("utf-8"),
|
||||
digestmod=hashlib.sha256,
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def fetch_points(sql: str, params: tuple) -> list[tuple[float, float]]:
|
||||
with get_conn() as conn:
|
||||
rows = conn.execute(sql, params).fetchall()
|
||||
return [(float(r["lat"]), float(r["lon"])) for r in rows]
|
||||
|
||||
|
||||
def nearest_from_db(
|
||||
lat: float, lon: float, table: str, type_col: str, value: str, fallback_points: list[tuple[float, float]]
|
||||
) -> tuple[int, bool]:
|
||||
rows = fetch_points(
|
||||
f"SELECT lat, lon FROM {table} WHERE {type_col} = ?",
|
||||
(value,),
|
||||
)
|
||||
if rows:
|
||||
return int(min(haversine_m(lat, lon, x, y) for x, y in rows)), False
|
||||
if fallback_points:
|
||||
return nearest_distance_m(lat, lon, fallback_points), True
|
||||
return 5000, True
|
||||
|
||||
|
||||
def area_from_db(lat: float, lon: float) -> str:
|
||||
with get_conn() as conn:
|
||||
rows = conn.execute("SELECT zone_type, lat, lon FROM osm_zone").fetchall()
|
||||
|
||||
if not rows:
|
||||
return classify_area(lat, lon)
|
||||
|
||||
best_type = "residential"
|
||||
best_dist = 999999.0
|
||||
for row in rows:
|
||||
dist = haversine_m(lat, lon, float(row["lat"]), float(row["lon"]))
|
||||
if dist < best_dist:
|
||||
best_dist = dist
|
||||
best_type = row["zone_type"]
|
||||
|
||||
return best_type if best_dist <= 500 else "residential"
|
||||
|
||||
|
||||
def road_from_db(lat: float, lon: float) -> str:
|
||||
with get_conn() as conn:
|
||||
rows = conn.execute("SELECT road_type, lat, lon FROM osm_road").fetchall()
|
||||
|
||||
if not rows:
|
||||
return classify_road(lat, lon)
|
||||
|
||||
best_type = "unknown"
|
||||
best_dist = 999999.0
|
||||
for row in rows:
|
||||
dist = haversine_m(lat, lon, float(row["lat"]), float(row["lon"]))
|
||||
if dist < best_dist:
|
||||
best_dist = dist
|
||||
best_type = row["road_type"]
|
||||
|
||||
return best_type if best_dist <= 300 else "unknown"
|
||||
|
||||
|
||||
def ensure_spot(lat: float, lon: float, now_iso: str) -> dict:
|
||||
s_id = spot_id_for(lat, lon)
|
||||
with get_conn() as conn:
|
||||
row = conn.execute("SELECT * FROM spot WHERE id = ?", (s_id,)).fetchone()
|
||||
if row:
|
||||
return dict(row)
|
||||
|
||||
area_type = area_from_db(lat, lon)
|
||||
road_type = road_from_db(lat, lon)
|
||||
|
||||
police_d, police_fallback = nearest_from_db(lat, lon, "osm_poi", "poi_type", "police", FALLBACK_POLICE_POINTS)
|
||||
fire_d, fire_fallback = nearest_from_db(lat, lon, "osm_poi", "poi_type", "fire", FALLBACK_FIRE_POINTS)
|
||||
hosp_d, hosp_fallback = nearest_from_db(lat, lon, "osm_poi", "poi_type", "hospital", FALLBACK_HOSPITAL_POINTS)
|
||||
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO spot (
|
||||
id, lat, lon, osm_area_type, road_type,
|
||||
distance_police_m, distance_fire_m, distance_hospital_m,
|
||||
created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(s_id, lat, lon, area_type, road_type, police_d, fire_d, hosp_d, now_iso, now_iso),
|
||||
)
|
||||
|
||||
return {
|
||||
"id": s_id,
|
||||
"lat": lat,
|
||||
"lon": lon,
|
||||
"osm_area_type": area_type,
|
||||
"road_type": road_type,
|
||||
"distance_police_m": police_d,
|
||||
"distance_fire_m": fire_d,
|
||||
"distance_hospital_m": hosp_d,
|
||||
"used_fallback_pois": police_fallback or fire_fallback or hosp_fallback,
|
||||
}
|
||||
|
||||
|
||||
def collect_local_event_factors(lat: float, lon: float, night_end: dt.datetime) -> list[dict]:
|
||||
next_morning_start = night_end
|
||||
next_morning_end = night_end + dt.timedelta(hours=4)
|
||||
|
||||
with get_conn() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT event_type, risk_modifier, source, lat, lon, start_datetime, end_datetime
|
||||
FROM open_data_event
|
||||
WHERE start_datetime <= ?
|
||||
AND end_datetime >= ?
|
||||
""",
|
||||
(to_iso(next_morning_end), to_iso(next_morning_start)),
|
||||
).fetchall()
|
||||
|
||||
factors = []
|
||||
seen = set()
|
||||
for row in rows:
|
||||
distance = haversine_m(lat, lon, float(row["lat"]), float(row["lon"]))
|
||||
if distance > 1000:
|
||||
continue
|
||||
|
||||
event_type = row["event_type"]
|
||||
label = {
|
||||
"waste": "Muellabfuhr am Morgen",
|
||||
"market": "Marktbetrieb am Morgen",
|
||||
"event": "Lokale Veranstaltung",
|
||||
"construction": "Baustelle",
|
||||
}.get(event_type, "Lokales Ereignis")
|
||||
dedupe_key = (
|
||||
event_type,
|
||||
row["risk_modifier"],
|
||||
row["start_datetime"] if "start_datetime" in row.keys() else "",
|
||||
row["end_datetime"] if "end_datetime" in row.keys() else "",
|
||||
round(float(row["lat"]), 4),
|
||||
round(float(row["lon"]), 4),
|
||||
)
|
||||
if dedupe_key in seen:
|
||||
continue
|
||||
seen.add(dedupe_key)
|
||||
|
||||
factors.append(
|
||||
{
|
||||
"key": f"event_{event_type}",
|
||||
"label": label,
|
||||
"points": float(row["risk_modifier"]),
|
||||
"source": row["source"],
|
||||
}
|
||||
)
|
||||
return factors
|
||||
|
||||
|
||||
def collect_community_factors(spot_id: str, at_time: dt.datetime) -> list[dict]:
|
||||
max_window = at_time - dt.timedelta(days=30)
|
||||
with get_conn() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT signal_type, timestamp
|
||||
FROM community_signal
|
||||
WHERE spot_id = ?
|
||||
AND timestamp >= ?
|
||||
""",
|
||||
(spot_id, to_iso(max_window)),
|
||||
).fetchall()
|
||||
|
||||
buckets = {
|
||||
"knock": {"base": -25.0, "half_life": 10.0, "window": 30, "label": "Klopfen gemeldet"},
|
||||
"noise": {"base": -15.0, "half_life": 7.0, "window": 14, "label": "Laerm gemeldet"},
|
||||
"calm": {"base": 10.0, "half_life": 7.0, "window": 14, "label": "Ruhig gemeldet"},
|
||||
"police": {"base": -18.0, "half_life": 10.0, "window": 30, "label": "Polizei-Einsatz gemeldet"},
|
||||
}
|
||||
|
||||
by_type: dict[str, float] = {key: 0.0 for key in buckets}
|
||||
for row in rows:
|
||||
signal_type = row["signal_type"]
|
||||
if signal_type not in buckets:
|
||||
continue
|
||||
cfg = buckets[signal_type]
|
||||
sig_ts = parse_iso8601(row["timestamp"])
|
||||
age_days = max(0.0, (at_time - sig_ts).total_seconds() / 86400.0)
|
||||
if age_days > cfg["window"]:
|
||||
continue
|
||||
by_type[signal_type] += cfg["base"] * decay(age_days, cfg["half_life"])
|
||||
|
||||
out = []
|
||||
for signal_type, points in by_type.items():
|
||||
if abs(points) < 0.5:
|
||||
continue
|
||||
label = buckets[signal_type]["label"]
|
||||
out.append(
|
||||
{
|
||||
"key": f"community_{signal_type}",
|
||||
"label": label,
|
||||
"points": points,
|
||||
"source": "community",
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def data_source_meta() -> list[dict]:
|
||||
with get_conn() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT source_name, imported_at, record_count, notes FROM data_source_state ORDER BY imported_at DESC"
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
|
||||
def source_health(sources: list[dict], now: dt.datetime) -> dict:
|
||||
if not sources:
|
||||
return {"freshest_age_hours": None, "stalest_age_hours": None, "stale_sources": [], "has_data": False}
|
||||
|
||||
ages = []
|
||||
stale_sources = []
|
||||
for src in sources:
|
||||
try:
|
||||
age_h = (now - parse_iso8601(src["imported_at"])).total_seconds() / 3600.0
|
||||
except Exception:
|
||||
continue
|
||||
age_h = max(0.0, age_h)
|
||||
ages.append((src["source_name"], age_h))
|
||||
if age_h > 24:
|
||||
stale_sources.append(src["source_name"])
|
||||
|
||||
if not ages:
|
||||
return {"freshest_age_hours": None, "stalest_age_hours": None, "stale_sources": [], "has_data": False}
|
||||
|
||||
freshest = min(age for _, age in ages)
|
||||
stalest = max(age for _, age in ages)
|
||||
return {
|
||||
"freshest_age_hours": round(freshest, 2),
|
||||
"stalest_age_hours": round(stalest, 2),
|
||||
"stale_sources": stale_sources,
|
||||
"has_data": True,
|
||||
}
|
||||
|
||||
|
||||
def compute_score_payload(lat: float, lon: float, at_time: dt.datetime) -> dict:
|
||||
now_iso = to_iso(utc_now())
|
||||
spot = ensure_spot(lat, lon, now_iso)
|
||||
sources = data_source_meta()
|
||||
health = source_health(sources, utc_now())
|
||||
|
||||
night_start, night_end = night_window_for(at_time)
|
||||
factors = []
|
||||
|
||||
area_factor = score_area_modifier(spot["osm_area_type"])
|
||||
factors.append({"key": area_factor.key, "label": area_factor.label, "points": area_factor.points, "source": "osm"})
|
||||
|
||||
if spot["distance_police_m"] < 200:
|
||||
factors.append({"key": "dist_police", "label": "Polizei in <200m", "points": -15.0, "source": "osm"})
|
||||
if spot["distance_hospital_m"] < 200:
|
||||
factors.append({"key": "dist_hospital", "label": "Krankenhaus in <200m", "points": -10.0, "source": "osm"})
|
||||
|
||||
if weekend_or_holiday(night_start):
|
||||
factors.append({"key": "time_weekend", "label": "Wochenende/Feiertag", "points": -10.0, "source": "time"})
|
||||
else:
|
||||
factors.append({"key": "time_weekday", "label": "Werktagnacht", "points": 5.0, "source": "time"})
|
||||
|
||||
factors.extend(collect_local_event_factors(lat, lon, night_end))
|
||||
factors.extend(collect_community_factors(spot["id"], at_time))
|
||||
|
||||
raw_score = 100.0 + sum(item["points"] for item in factors)
|
||||
score = clamp_score(raw_score)
|
||||
|
||||
top_reasons = sorted(factors, key=lambda item: abs(item["points"]), reverse=True)[:4]
|
||||
reasons = [f"{item['label']} ({item['points']:+.0f})" for item in top_reasons]
|
||||
|
||||
return {
|
||||
"spot_id": spot["id"],
|
||||
"score": score,
|
||||
"ampel": ampel(score),
|
||||
"reasons": reasons,
|
||||
"factors": top_reasons,
|
||||
"night_window": {
|
||||
"start": to_iso(night_start),
|
||||
"end": to_iso(night_end),
|
||||
},
|
||||
"meta": {
|
||||
"data_updated_at": now_iso,
|
||||
"region": "DE-NW (Pilot: Kreis Mettmann)",
|
||||
"attribution": "Kartendaten: OpenStreetMap-Mitwirkende (ODbL)",
|
||||
"sources": sources,
|
||||
"health": health,
|
||||
"used_fallback_pois": bool(spot.get("used_fallback_pois", False)),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def handle_score(handler: BaseHTTPRequestHandler, query: dict[str, list[str]]) -> None:
|
||||
try:
|
||||
lat = float(query.get("lat", [""])[0])
|
||||
lon = float(query.get("lon", [""])[0])
|
||||
at = parse_iso8601(query.get("at", [""])[0])
|
||||
except Exception:
|
||||
json_response(handler, HTTPStatus.BAD_REQUEST, {"error": "invalid_query"})
|
||||
return
|
||||
|
||||
if not (47.0 <= lat <= 55.5 and 5.0 <= lon <= 16.0):
|
||||
json_response(handler, HTTPStatus.BAD_REQUEST, {"error": "lat_lon_out_of_bounds"})
|
||||
return
|
||||
|
||||
payload = compute_score_payload(lat, lon, at)
|
||||
json_response(handler, HTTPStatus.OK, payload)
|
||||
|
||||
|
||||
def handle_signal(handler: BaseHTTPRequestHandler) -> None:
|
||||
try:
|
||||
body = read_json(handler)
|
||||
except Exception:
|
||||
json_response(handler, HTTPStatus.BAD_REQUEST, {"error": "invalid_json"})
|
||||
return
|
||||
|
||||
spot_id = body.get("spot_id")
|
||||
signal_type = body.get("signal_type")
|
||||
device_token = body.get("device_token")
|
||||
timestamp = parse_iso8601(body.get("timestamp"))
|
||||
|
||||
if not isinstance(spot_id, str) or not spot_id:
|
||||
json_response(handler, HTTPStatus.BAD_REQUEST, {"error": "spot_id_required"})
|
||||
return
|
||||
if signal_type not in {"calm", "noise", "knock", "police"}:
|
||||
json_response(handler, HTTPStatus.BAD_REQUEST, {"error": "invalid_signal_type"})
|
||||
return
|
||||
if not isinstance(device_token, str) or len(device_token) < 16:
|
||||
json_response(handler, HTTPStatus.BAD_REQUEST, {"error": "invalid_device_token"})
|
||||
return
|
||||
|
||||
hashed = hashed_device(device_token)
|
||||
now = utc_now()
|
||||
if timestamp > now + dt.timedelta(minutes=5):
|
||||
timestamp = now
|
||||
|
||||
cooldown_start = timestamp - dt.timedelta(hours=SIGNAL_COOLDOWN_HOURS)
|
||||
|
||||
with get_conn() as conn:
|
||||
spot_exists = conn.execute("SELECT 1 FROM spot WHERE id = ?", (spot_id,)).fetchone()
|
||||
if not spot_exists:
|
||||
json_response(handler, HTTPStatus.BAD_REQUEST, {"error": "unknown_spot_id"})
|
||||
return
|
||||
|
||||
latest = conn.execute(
|
||||
"""
|
||||
SELECT timestamp
|
||||
FROM community_signal
|
||||
WHERE spot_id = ?
|
||||
AND hashed_device = ?
|
||||
AND timestamp >= ?
|
||||
ORDER BY timestamp DESC
|
||||
LIMIT 1
|
||||
""",
|
||||
(spot_id, hashed, to_iso(cooldown_start)),
|
||||
).fetchone()
|
||||
|
||||
if latest:
|
||||
next_allowed = parse_iso8601(latest["timestamp"]) + dt.timedelta(hours=SIGNAL_COOLDOWN_HOURS)
|
||||
json_response(
|
||||
handler,
|
||||
HTTPStatus.TOO_MANY_REQUESTS,
|
||||
{
|
||||
"accepted": False,
|
||||
"error": "cooldown_active",
|
||||
"next_allowed_at": to_iso(next_allowed),
|
||||
},
|
||||
)
|
||||
return
|
||||
|
||||
day_bucket = timestamp.date().isoformat()
|
||||
try:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO community_signal (
|
||||
id, spot_id, signal_type, hashed_device, timestamp, day_bucket
|
||||
) VALUES (?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
str(uuid.uuid4()),
|
||||
spot_id,
|
||||
signal_type,
|
||||
hashed,
|
||||
to_iso(timestamp),
|
||||
day_bucket,
|
||||
),
|
||||
)
|
||||
except Exception:
|
||||
json_response(
|
||||
handler,
|
||||
HTTPStatus.TOO_MANY_REQUESTS,
|
||||
{
|
||||
"accepted": False,
|
||||
"error": "daily_limit",
|
||||
},
|
||||
)
|
||||
return
|
||||
|
||||
json_response(
|
||||
handler,
|
||||
HTTPStatus.CREATED,
|
||||
{
|
||||
"accepted": True,
|
||||
"cooldown_hours": SIGNAL_COOLDOWN_HOURS,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class StaySenseHandler(BaseHTTPRequestHandler):
|
||||
def log_message(self, format: str, *args) -> None:
|
||||
return
|
||||
|
||||
def do_GET(self) -> None:
|
||||
parsed = urlparse(self.path)
|
||||
if parsed.path == "/health":
|
||||
sources = data_source_meta()
|
||||
json_response(self, HTTPStatus.OK, {"status": "ok", "sources": sources, "health": source_health(sources, utc_now())})
|
||||
return
|
||||
if parsed.path == "/spot/score":
|
||||
query = parse_qs(parsed.query)
|
||||
handle_score(self, query)
|
||||
return
|
||||
json_response(self, HTTPStatus.NOT_FOUND, {"error": "not_found"})
|
||||
|
||||
def do_POST(self) -> None:
|
||||
parsed = urlparse(self.path)
|
||||
if parsed.path == "/spot/signal":
|
||||
handle_signal(self)
|
||||
return
|
||||
json_response(self, HTTPStatus.NOT_FOUND, {"error": "not_found"})
|
||||
|
||||
|
||||
def main() -> None:
|
||||
init_db()
|
||||
server = ThreadingHTTPServer((HOST, PORT), StaySenseHandler)
|
||||
print(f"StaySense API listening on http://{HOST}:{PORT}")
|
||||
server.serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
21
backend/tests/test_open_data_connector.py
Normal file
21
backend/tests/test_open_data_connector.py
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from open_data_connector import import_from_config
|
||||
|
||||
|
||||
class OpenDataConnectorTests(unittest.TestCase):
|
||||
def test_import_from_config_has_stats(self) -> None:
|
||||
cfg = Path(__file__).resolve().parents[2] / "docs" / "open_data_sources.json"
|
||||
result = import_from_config(cfg, prune_legacy=True)
|
||||
self.assertIn("imported", result)
|
||||
self.assertIn("pruned", result)
|
||||
self.assertGreaterEqual(len(result["imported"]), 1)
|
||||
|
||||
imported = result["imported"][0]
|
||||
self.assertIn("stats", imported)
|
||||
self.assertIn("accepted", imported["stats"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
25
backend/tests/test_score_engine.py
Normal file
25
backend/tests/test_score_engine.py
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
import datetime as dt
|
||||
import unittest
|
||||
|
||||
from score_engine import ampel, clamp_score, night_window_for
|
||||
|
||||
|
||||
class ScoreEngineTests(unittest.TestCase):
|
||||
def test_ampel_thresholds(self) -> None:
|
||||
self.assertEqual(ampel(70), "green")
|
||||
self.assertEqual(ampel(45), "yellow")
|
||||
self.assertEqual(ampel(10), "red")
|
||||
|
||||
def test_clamp_score(self) -> None:
|
||||
self.assertEqual(clamp_score(-20), 0)
|
||||
self.assertEqual(clamp_score(120), 100)
|
||||
|
||||
def test_night_window(self) -> None:
|
||||
ref = dt.datetime(2026, 2, 15, 23, 30, tzinfo=dt.timezone.utc)
|
||||
start, end = night_window_for(ref)
|
||||
self.assertEqual(start.hour, 22)
|
||||
self.assertEqual(end.hour, 6)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue