diff --git a/backend/app/categorize.py b/backend/app/categorize.py index f96b4f4..ec33e80 100644 --- a/backend/app/categorize.py +++ b/backend/app/categorize.py @@ -40,8 +40,11 @@ _RULES: tuple[tuple[str, tuple[tuple[str, int], ...]], ...] = ( ("rabatt*", 3), ("schnaeppchen*", 3), ("prime day", 3), ("black friday", 3), ("gutschein*", 3), ("sparangebot*", 3), ("top-angebot*", 3), ("sommerangebot*", 3), ("knaller", 3), ("ausverkauf", 3), ("tiefpreis*", 3), ("rekordpreis*", 3), - ("preissturz", 3), ("bestpreis*", 3), ("deal", 2), ("deals", 2), ("lidl", 2), - ("aldi", 2), ("reduziert", 2), ("sale", 1), ("prozent auf", 3), ("guenstiger*", 1), + ("preissturz", 3), ("bestpreis*", 3), ("deal", 2), ("deals", 2), + # Lidl and Aldi articles are always discount posts on this blog, so they + # carry enough weight to trigger the override below on their own. + ("lidl", 3), ("aldi", 3), ("im angebot", 3), + ("reduziert", 2), ("sale", 1), ("prozent auf", 3), ("guenstiger*", 1), )), ("in-eigener-sache", ( ("vanityontour", 3), ("vanitycast", 3), ("expense logbook", 3), @@ -96,6 +99,8 @@ _RULES: tuple[tuple[str, tuple[tuple[str, int], ...]], ...] = ( ("kassettentoilette*", 3), ("toilette*", 2), ("gaswarner", 3), ("rauchmelder", 3), ("feuerloescher", 3), ("co melder", 3), ("router", 3), ("mobilfunk", 3), ("wlan", 3), ("lte", 3), ("5g", 3), ("internet", 2), ("buchtipp*", 2), + ("duschzelt*", 3), ("kuppelzelt*", 3), ("pop-up-zelt*", 3), ("messer", 3), + ("stuhl", 3), ("faltstuhl*", 3), ("klappstuhl*", 3), )), ("campingplaetze", ( ("campingplatz*", 3), ("campingplaetze", 3), ("5-sterne*", 3), ("fuenf sterne", 3), @@ -145,7 +150,11 @@ _RULES: tuple[tuple[str, tuple[tuple[str, int], ...]], ...] = ( ("rundreise*", 3), ("roadtrip*", 3), ("reisebericht*", 3), ("ausflugsziel*", 3), ("kurztrip*", 3), ("europa", 2), ("kueste", 2), ("region", 1), ("uckermark", 3), ("lausitz", 3), ("schwaebische alb", 3), ("ausflug*", 2), ("geheimtipp*", 2), - ("uebersee", 2), ("kanada", 3), + ("uebersee", 2), ("kanada", 3), ("pfalz", 1), ("erzgebirge", 2), + ("spreewald", 2), ("vogtland", 2), ("rhoen", 2), ("odenwald", 2), + ("chiemsee", 2), ("ammersee", 2), ("muensterland", 2), ("ostfriesland", 2), + ("emsland", 2), ("altmuehltal", 2), ("teutoburger wald", 2), ("taunus", 2), + ("westerwald", 2), ("bergisches land", 2), ("nordwesten", 2), )), ("camping-tipps", ( ("tipp", 3), ("tipps", 3), ("tricks", 3), ("ratgeber", 3), ("anleitung", 3), diff --git a/backend/app/config.py b/backend/app/config.py index 1a92e0c..a0c8f66 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -30,6 +30,11 @@ class Settings(BaseSettings): wordpress_username: str | None = Field(default=None, validation_alias=AliasChoices("WORDPRESS_USERNAME", "WP_USERNAME")) wordpress_app_password: str | None = Field(default=None, validation_alias=AliasChoices("WORDPRESS_APP_PASSWORD", "WP_PASSWORD")) wordpress_default_status: str = "draft" + # Tag hygiene: the rewriter proposes far more tags than a post needs, and + # every unknown one used to be created on the spot - that is how 955 posts + # accumulated 3.055 tags, 1.683 of them used exactly once. + wordpress_max_tags_per_post: int = 5 + wordpress_new_tag_min_proposals: int = 3 openai_api_key: str | None = Field(default=None, validation_alias=AliasChoices("OPENAI_API_KEY")) openai_model: str = "gpt-4o-mini" diff --git a/backend/app/db.py b/backend/app/db.py index b6ef898..6aa1e0f 100644 --- a/backend/app/db.py +++ b/backend/app/db.py @@ -118,6 +118,21 @@ def init_db() -> None: UNIQUE(source_url) ); + -- One row per (tag, article) the rewriter ever proposed, whether or + -- not the tag made it into WordPress. The number of rows per + -- name_key is the "is this tag established" signal that gates + -- creating the tag in WordPress. + CREATE TABLE IF NOT EXISTS tag_proposals ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name_key TEXT NOT NULL, + name TEXT NOT NULL, + article_id INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE(name_key, article_id) + ); + + CREATE INDEX IF NOT EXISTS idx_tag_proposals_name_key ON tag_proposals(name_key); + CREATE INDEX IF NOT EXISTS idx_articles_source_article_id ON articles(source_article_id); CREATE INDEX IF NOT EXISTS idx_articles_source_hash ON articles(source_hash); CREATE UNIQUE INDEX IF NOT EXISTS uq_articles_feed_source_article_id diff --git a/backend/app/repositories.py b/backend/app/repositories.py index cf38055..8032077 100644 --- a/backend/app/repositories.py +++ b/backend/app/repositories.py @@ -853,3 +853,41 @@ def list_articles(limit: int = 100, status_filter: str | None = None) -> list[di (safe_limit,), ).fetchall() return rows_to_dicts(rows) + + +def record_tag_proposals(names: list[str], article_id: int | None) -> dict[str, int]: + """Record that these tags were proposed for one article. + + Returns, per casefolded name, how many distinct articles have proposed it so + far - this one included. That count is what decides whether a tag is + established enough to be created in WordPress: a tag the rewriter invents + once for a single article should not become a permanent taxonomy entry. + + Re-publishing an article does not inflate the count, because the row is + keyed on (name_key, article_id). Articles without an id (ad-hoc publishing) + all share the sentinel 0 and therefore count once in total. + """ + counts: dict[str, int] = {} + if not names: + return counts + key_for_article = int(article_id or 0) + with get_conn() as conn: + for raw in names: + name = str(raw or "").strip() + if not name: + continue + name_key = name.casefold() + conn.execute( + """ + INSERT INTO tag_proposals (name_key, name, article_id) + VALUES (?, ?, ?) + ON CONFLICT(name_key, article_id) DO NOTHING + """, + (name_key, name, key_for_article), + ) + row = conn.execute( + "SELECT COUNT(*) AS n FROM tag_proposals WHERE name_key = ?", + (name_key,), + ).fetchone() + counts[name_key] = int(row["n"] if row else 0) + return counts diff --git a/backend/app/wordpress.py b/backend/app/wordpress.py index 0d81ae1..32ec594 100644 --- a/backend/app/wordpress.py +++ b/backend/app/wordpress.py @@ -13,6 +13,7 @@ from urllib.parse import quote_plus, urlparse from urllib.request import Request, urlopen from . import categorize +from . import repositories from .config import get_settings @@ -91,7 +92,25 @@ def _selected_tags_from_meta(meta_json: str | None) -> list[str]: return tags -def _resolve_wp_tag_ids(*, base_url: str, auth_header: str, tags: list[str]) -> list[int]: +def _resolve_wp_tag_ids( + *, + base_url: str, + auth_header: str, + tags: list[str], + creatable: set[str], +) -> list[int]: + """Map tag names to WordPress tag IDs, creating only established ones. + + A name matches an existing tag only on an exact (case-insensitive) name. + The former fallback of taking the first search hit filed "Camping" under + the unrelated existing tag "Campingplatz" whenever the exact tag was + missing. + + `creatable` holds the casefolded names that cleared the proposal threshold + and may be created in WordPress. Everything else is dropped with a log line + rather than turned into a new tag - an empty set means this run adds nothing + to the taxonomy. + """ ids: list[int] = [] seen: set[int] = set() for tag in tags: @@ -113,12 +132,12 @@ def _resolve_wp_tag_ids(*, base_url: str, auth_header: str, tags: list[str]) -> if row_name.casefold() == name.casefold(): tag_id = rid break - if tag_id is None: - for row in result: - if isinstance(row, dict) and int(row.get("id", 0) or 0) > 0: - tag_id = int(row.get("id", 0)) - break if tag_id is None: + if name.casefold() not in creatable: + _logger.info( + "Schlagwort '%s' noch nicht etabliert - wird nicht angelegt", name + ) + continue created = _wp_request( base_url=base_url, auth_header=auth_header, @@ -130,6 +149,7 @@ def _resolve_wp_tag_ids(*, base_url: str, auth_header: str, tags: list[str]) -> rid = int(created.get("id", 0) or 0) if rid > 0: tag_id = rid + _logger.info("Schlagwort '%s' in WordPress neu angelegt (#%s)", name, rid) if tag_id is not None and tag_id > 0 and tag_id not in seen: seen.add(tag_id) ids.append(tag_id) @@ -138,6 +158,32 @@ def _resolve_wp_tag_ids(*, base_url: str, auth_header: str, tags: list[str]) -> return ids +def _creatable_tag_names(tags: list[str], article_id: Any) -> set[str]: + """Casefolded names that may be created as new WordPress tags. + + A proposed tag has to show up for `wordpress_new_tag_min_proposals` + different articles before it earns a taxonomy entry. Until then it is + counted and dropped, so one-off inventions stop accumulating. + + If the bookkeeping itself fails, nothing is creatable: existing tags are + still assigned, and the taxonomy simply does not grow that run. + """ + if not tags: + return set() + settings = get_settings() + threshold = max(1, settings.wordpress_new_tag_min_proposals) + try: + aid = int(article_id) if article_id is not None else None + except (TypeError, ValueError): + aid = None + try: + counts = repositories.record_tag_proposals(tags, aid) + except Exception as exc: + _logger.warning("Schlagwort-Zähler nicht verfügbar, lege keine neuen an: %s", exc) + return set() + return {name for name, seen in counts.items() if seen >= threshold} + + _category_id_cache: dict[str, int | None] = {} @@ -552,10 +598,14 @@ def publish_article_draft(article: dict[str, Any]) -> tuple[int, str | None]: wp_post_id = article.get("wp_post_id") tag_names = _selected_tags_from_meta(article.get("meta_json")) + # Only the leading few tags reach WordPress; the full list still feeds the + # category rules below, which were validated against it. + wp_tag_names = tag_names[: max(0, settings.wordpress_max_tags_per_post)] tag_ids = _resolve_wp_tag_ids( base_url=settings.wordpress_base_url, auth_header=auth, - tags=tag_names, + tags=wp_tag_names, + creatable=_creatable_tag_names(wp_tag_names, article.get("id")), ) if tag_ids: payload["tags"] = tag_ids diff --git a/backend/tests/test_wordpress.py b/backend/tests/test_wordpress.py index 6ac591e..2f77c65 100644 --- a/backend/tests/test_wordpress.py +++ b/backend/tests/test_wordpress.py @@ -1,9 +1,12 @@ import os +import tempfile import unittest +from pathlib import Path from unittest.mock import patch from backend.app import config as config_module from backend.app import wordpress as wordpress_module +from backend.app.db import init_db from backend.app.wordpress import publish_article_draft @@ -12,15 +15,28 @@ class TestWordpressPublish(unittest.TestCase): os.environ["WORDPRESS_BASE_URL"] = "https://example.org" os.environ["WORDPRESS_USERNAME"] = "wp-user" os.environ["WORDPRESS_APP_PASSWORD"] = "wp-pass" + # Publishing records tag proposals, so it needs a database of its own - + # otherwise the tests would write into the live one. + self.tmp_dir = tempfile.TemporaryDirectory() + os.environ["APP_DB_PATH"] = str(Path(self.tmp_dir.name) / "test.db") config_module.get_settings.cache_clear() + init_db() # The category lookup is cached for the process lifetime; without this # one test would resolve a slug that the next one expects to be missing. wordpress_module._category_id_cache.clear() def tearDown(self) -> None: - for key in ("WORDPRESS_BASE_URL", "WORDPRESS_USERNAME", "WORDPRESS_APP_PASSWORD"): + for key in ( + "WORDPRESS_BASE_URL", + "WORDPRESS_USERNAME", + "WORDPRESS_APP_PASSWORD", + "APP_DB_PATH", + "WORDPRESS_MAX_TAGS_PER_POST", + "WORDPRESS_NEW_TAG_MIN_PROPOSALS", + ): os.environ.pop(key, None) config_module.get_settings.cache_clear() + self.tmp_dir.cleanup() @patch("backend.app.wordpress._upload_featured_media") @patch("backend.app.wordpress._wp_request") @@ -87,7 +103,7 @@ class TestWordpressPublish(unittest.TestCase): @patch("backend.app.wordpress._upload_featured_media") @patch("backend.app.wordpress._wp_request") - def test_publish_resolves_and_sets_tags(self, mock_wp_request, mock_upload_media) -> None: + def test_publish_resolves_existing_tags_and_skips_unknown_ones(self, mock_wp_request, mock_upload_media) -> None: def _fake_wp_request(**kwargs): endpoint = kwargs.get("endpoint", "") method = kwargs.get("method", "") @@ -95,17 +111,13 @@ class TestWordpressPublish(unittest.TestCase): if "Rheingas" in endpoint: return [{"id": 11, "name": "Rheingas"}] return [] - if method == "POST" and endpoint == "tags": - name = (kwargs.get("payload") or {}).get("name") - if name == "Gasflasche": - return {"id": 12, "name": "Gasflasche"} - return {"id": 13, "name": str(name)} if method == "POST" and endpoint == "posts": return {"id": 900, "link": "https://example.org/?p=900"} return {} mock_wp_request.side_effect = _fake_wp_request article = { + "id": 1, "title": "Tag Test", "content_raw": "Inhalt", "source_url": "https://example.com/source", @@ -117,7 +129,138 @@ class TestWordpressPublish(unittest.TestCase): post_calls = [call for call in mock_wp_request.call_args_list if call.kwargs.get("endpoint") == "posts"] self.assertEqual(len(post_calls), 1) payload = post_calls[0].kwargs.get("payload", {}) - self.assertEqual(payload.get("tags"), [11, 12]) + # "Gasflasche" was proposed for the first time and is not created yet. + self.assertEqual(payload.get("tags"), [11]) + self.assertFalse( + any(c.kwargs.get("method") == "POST" and c.kwargs.get("endpoint") == "tags" + for c in mock_wp_request.call_args_list) + ) + + @patch("backend.app.wordpress._upload_featured_media") + @patch("backend.app.wordpress._wp_request") + def test_new_tag_is_created_once_enough_articles_proposed_it(self, mock_wp_request, mock_upload_media) -> None: + created_names: list[str] = [] + + def _fake_wp_request(**kwargs): + endpoint = kwargs.get("endpoint", "") + method = kwargs.get("method", "") + if method == "GET" and endpoint.startswith("tags?search="): + return [] + if method == "POST" and endpoint == "tags": + created_names.append(str((kwargs.get("payload") or {}).get("name"))) + return {"id": 55, "name": "Wintercamping"} + if method == "POST" and endpoint == "posts": + return {"id": 910, "link": "https://example.org/?p=910"} + return {} + + mock_wp_request.side_effect = _fake_wp_request + + def _article(article_id: int) -> dict: + return { + "id": article_id, + "title": f"Artikel {article_id}", + "content_raw": "Inhalt", + "source_url": f"https://example.com/source/{article_id}", + "canonical_url": f"https://example.com/source/{article_id}", + "meta_json": '{"generated_tags":["Wintercamping"]}', + } + + for article_id in (1, 2): + publish_article_draft(_article(article_id)) + self.assertEqual(created_names, []) + + publish_article_draft(_article(3)) + self.assertEqual(created_names, ["Wintercamping"]) + payload = [c for c in mock_wp_request.call_args_list if c.kwargs.get("endpoint") == "posts"][-1].kwargs["payload"] + self.assertEqual(payload.get("tags"), [55]) + + @patch("backend.app.wordpress._upload_featured_media") + @patch("backend.app.wordpress._wp_request") + def test_republishing_the_same_article_does_not_count_twice(self, mock_wp_request, mock_upload_media) -> None: + def _fake_wp_request(**kwargs): + endpoint = kwargs.get("endpoint", "") + method = kwargs.get("method", "") + if method == "GET" and endpoint.startswith("tags?search="): + return [] + if method == "POST" and endpoint == "tags": + return {"id": 66, "name": "Solaranlage"} + if method == "POST": + return {"id": 920, "link": "https://example.org/?p=920"} + return {} + + mock_wp_request.side_effect = _fake_wp_request + article = { + "id": 7, + "title": "Immer wieder derselbe Artikel", + "content_raw": "Inhalt", + "source_url": "https://example.com/source", + "canonical_url": "https://example.com/source", + "meta_json": '{"generated_tags":["Solaranlage"]}', + } + for _ in range(4): + publish_article_draft(article) + self.assertFalse( + any(c.kwargs.get("method") == "POST" and c.kwargs.get("endpoint") == "tags" + for c in mock_wp_request.call_args_list) + ) + + @patch("backend.app.wordpress._upload_featured_media") + @patch("backend.app.wordpress._wp_request") + def test_publish_caps_the_number_of_tags_per_post(self, mock_wp_request, mock_upload_media) -> None: + looked_up: list[str] = [] + + def _fake_wp_request(**kwargs): + endpoint = kwargs.get("endpoint", "") + method = kwargs.get("method", "") + if method == "GET" and endpoint.startswith("tags?search="): + looked_up.append(endpoint) + return [] + if method == "POST" and endpoint == "posts": + return {"id": 930, "link": "https://example.org/?p=930"} + return {} + + mock_wp_request.side_effect = _fake_wp_request + names = [f"Tag{i}" for i in range(1, 13)] + article = { + "id": 42, + "title": "Viele Schlagwörter", + "content_raw": "Inhalt", + "source_url": "https://example.com/source", + "canonical_url": "https://example.com/source", + "meta_json": '{"generated_tags":' + str(names).replace("'", '"') + "}", + } + publish_article_draft(article) + self.assertEqual(len(looked_up), 5) + + @patch("backend.app.wordpress._upload_featured_media") + @patch("backend.app.wordpress._wp_request") + def test_tag_lookup_ignores_near_misses(self, mock_wp_request, mock_upload_media) -> None: + """A search hit that is not the exact name must not be reused. + + Plain substring reuse filed "Camping" under the unrelated existing tag + "Campingplatz" whenever the exact tag was missing. + """ + def _fake_wp_request(**kwargs): + endpoint = kwargs.get("endpoint", "") + method = kwargs.get("method", "") + if method == "GET" and endpoint.startswith("tags?search="): + return [{"id": 99, "name": "Campingplatz"}] + if method == "POST" and endpoint == "posts": + return {"id": 940, "link": "https://example.org/?p=940"} + return {} + + mock_wp_request.side_effect = _fake_wp_request + article = { + "id": 5, + "title": "Naher Treffer", + "content_raw": "Inhalt", + "source_url": "https://example.com/source", + "canonical_url": "https://example.com/source", + "meta_json": '{"generated_tags":["Camping"]}', + } + publish_article_draft(article) + payload = [c for c in mock_wp_request.call_args_list if c.kwargs.get("endpoint") == "posts"][0].kwargs["payload"] + self.assertNotIn("tags", payload) @patch("backend.app.wordpress._upload_featured_media") @patch("backend.app.wordpress._wp_request")