feat(wordpress): only assign tags that are established

_resolve_wp_tag_ids created a WordPress tag for every keyword the rewriter
invented, up to 12 per post. That is where the 3.055 tags for 955 posts
came from - 1.683 of them used exactly once, 473 attached to no post at
all. The categories are stable now, but the tags would simply grow back.

Three changes, all on the write path:

A proposed tag has to appear for wordpress_new_tag_min_proposals (3)
different articles before it is created. Proposals are counted in the new
tag_proposals table, keyed on (name, article), so re-publishing an article
does not inflate its own count. Tags that already exist in WordPress are
assigned as before - the gate only guards creation.

Only the first wordpress_max_tags_per_post (5) tags reach WordPress. The
full list still feeds the category rules, which were validated against it.

The lookup fallback of reusing the first search hit is gone. It filed
"Camping" under the unrelated existing tag "Campingplatz" whenever the
exact tag was missing, which quietly produced wrong tags rather than none.

If the proposal bookkeeping fails, nothing is creatable that run: existing
tags still get assigned and the taxonomy stays put, rather than falling
back to creating everything.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

backend/app/categorize.py

@@ -40,8 +40,11 @@ _RULES: tuple[tuple[str, tuple[tuple[str, int], ...]], ...] = (
         ("rabatt*", 3), ("schnaeppchen*", 3), ("prime day", 3), ("black friday", 3),
         ("gutschein*", 3), ("sparangebot*", 3), ("top-angebot*", 3), ("sommerangebot*", 3),
         ("knaller", 3), ("ausverkauf", 3), ("tiefpreis*", 3), ("rekordpreis*", 3),
-        ("preissturz", 3), ("bestpreis*", 3), ("deal", 2), ("deals", 2), ("lidl", 2),
-        ("aldi", 2), ("reduziert", 2), ("sale", 1), ("prozent auf", 3), ("guenstiger*", 1),
+        ("preissturz", 3), ("bestpreis*", 3), ("deal", 2), ("deals", 2),
+        # Lidl and Aldi articles are always discount posts on this blog, so they
+        # carry enough weight to trigger the override below on their own.
+        ("lidl", 3), ("aldi", 3), ("im angebot", 3),
+        ("reduziert", 2), ("sale", 1), ("prozent auf", 3), ("guenstiger*", 1),
     )),
     ("in-eigener-sache", (
         ("vanityontour", 3), ("vanitycast", 3), ("expense logbook", 3),
@@ -96,6 +99,8 @@ _RULES: tuple[tuple[str, tuple[tuple[str, int], ...]], ...] = (
         ("kassettentoilette*", 3), ("toilette*", 2), ("gaswarner", 3), ("rauchmelder", 3),
         ("feuerloescher", 3), ("co melder", 3), ("router", 3), ("mobilfunk", 3), ("wlan", 3),
         ("lte", 3), ("5g", 3), ("internet", 2), ("buchtipp*", 2),
+        ("duschzelt*", 3), ("kuppelzelt*", 3), ("pop-up-zelt*", 3), ("messer", 3),
+        ("stuhl", 3), ("faltstuhl*", 3), ("klappstuhl*", 3),
     )),
     ("campingplaetze", (
         ("campingplatz*", 3), ("campingplaetze", 3), ("5-sterne*", 3), ("fuenf sterne", 3),
@@ -145,7 +150,11 @@ _RULES: tuple[tuple[str, tuple[tuple[str, int], ...]], ...] = (
         ("rundreise*", 3), ("roadtrip*", 3), ("reisebericht*", 3), ("ausflugsziel*", 3),
         ("kurztrip*", 3), ("europa", 2), ("kueste", 2), ("region", 1), ("uckermark", 3),
         ("lausitz", 3), ("schwaebische alb", 3), ("ausflug*", 2), ("geheimtipp*", 2),
-        ("uebersee", 2), ("kanada", 3),
+        ("uebersee", 2), ("kanada", 3), ("pfalz", 1), ("erzgebirge", 2),
+        ("spreewald", 2), ("vogtland", 2), ("rhoen", 2), ("odenwald", 2),
+        ("chiemsee", 2), ("ammersee", 2), ("muensterland", 2), ("ostfriesland", 2),
+        ("emsland", 2), ("altmuehltal", 2), ("teutoburger wald", 2), ("taunus", 2),
+        ("westerwald", 2), ("bergisches land", 2), ("nordwesten", 2),
     )),
     ("camping-tipps", (
         ("tipp", 3), ("tipps", 3), ("tricks", 3), ("ratgeber", 3), ("anleitung", 3),

backend/app/config.py

@@ -30,6 +30,11 @@ class Settings(BaseSettings):
     wordpress_username: str | None = Field(default=None, validation_alias=AliasChoices("WORDPRESS_USERNAME", "WP_USERNAME"))
     wordpress_app_password: str | None = Field(default=None, validation_alias=AliasChoices("WORDPRESS_APP_PASSWORD", "WP_PASSWORD"))
     wordpress_default_status: str = "draft"
+    # Tag hygiene: the rewriter proposes far more tags than a post needs, and
+    # every unknown one used to be created on the spot - that is how 955 posts
+    # accumulated 3.055 tags, 1.683 of them used exactly once.
+    wordpress_max_tags_per_post: int = 5
+    wordpress_new_tag_min_proposals: int = 3
     openai_api_key: str | None = Field(default=None, validation_alias=AliasChoices("OPENAI_API_KEY"))
     openai_model: str = "gpt-4o-mini"
 

backend/app/db.py

@@ -118,6 +118,21 @@ def init_db() -> None:
                 UNIQUE(source_url)
             );
 
+            -- One row per (tag, article) the rewriter ever proposed, whether or
+            -- not the tag made it into WordPress. The number of rows per
+            -- name_key is the "is this tag established" signal that gates
+            -- creating the tag in WordPress.
+            CREATE TABLE IF NOT EXISTS tag_proposals (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                name_key TEXT NOT NULL,
+                name TEXT NOT NULL,
+                article_id INTEGER NOT NULL DEFAULT 0,
+                created_at TEXT NOT NULL DEFAULT (datetime('now')),
+                UNIQUE(name_key, article_id)
+            );
+
+            CREATE INDEX IF NOT EXISTS idx_tag_proposals_name_key ON tag_proposals(name_key);
+
             CREATE INDEX IF NOT EXISTS idx_articles_source_article_id ON articles(source_article_id);
             CREATE INDEX IF NOT EXISTS idx_articles_source_hash ON articles(source_hash);
             CREATE UNIQUE INDEX IF NOT EXISTS uq_articles_feed_source_article_id

backend/app/repositories.py

@@ -853,3 +853,41 @@ def list_articles(limit: int = 100, status_filter: str | None = None) -> list[di
                 (safe_limit,),
             ).fetchall()
     return rows_to_dicts(rows)
+
+
+def record_tag_proposals(names: list[str], article_id: int | None) -> dict[str, int]:
+    """Record that these tags were proposed for one article.
+
+    Returns, per casefolded name, how many distinct articles have proposed it so
+    far - this one included. That count is what decides whether a tag is
+    established enough to be created in WordPress: a tag the rewriter invents
+    once for a single article should not become a permanent taxonomy entry.
+
+    Re-publishing an article does not inflate the count, because the row is
+    keyed on (name_key, article_id). Articles without an id (ad-hoc publishing)
+    all share the sentinel 0 and therefore count once in total.
+    """
+    counts: dict[str, int] = {}
+    if not names:
+        return counts
+    key_for_article = int(article_id or 0)
+    with get_conn() as conn:
+        for raw in names:
+            name = str(raw or "").strip()
+            if not name:
+                continue
+            name_key = name.casefold()
+            conn.execute(
+                """
+                INSERT INTO tag_proposals (name_key, name, article_id)
+                VALUES (?, ?, ?)
+                ON CONFLICT(name_key, article_id) DO NOTHING
+                """,
+                (name_key, name, key_for_article),
+            )
+            row = conn.execute(
+                "SELECT COUNT(*) AS n FROM tag_proposals WHERE name_key = ?",
+                (name_key,),
+            ).fetchone()
+            counts[name_key] = int(row["n"] if row else 0)
+    return counts

backend/app/wordpress.py

@@ -13,6 +13,7 @@ from urllib.parse import quote_plus, urlparse
 from urllib.request import Request, urlopen
 
 from . import categorize
+from . import repositories
 from .config import get_settings
 
 
@@ -91,7 +92,25 @@ def _selected_tags_from_meta(meta_json: str | None) -> list[str]:
     return tags
 
 
-def _resolve_wp_tag_ids(*, base_url: str, auth_header: str, tags: list[str]) -> list[int]:
+def _resolve_wp_tag_ids(
+    *,
+    base_url: str,
+    auth_header: str,
+    tags: list[str],
+    creatable: set[str],
+) -> list[int]:
+    """Map tag names to WordPress tag IDs, creating only established ones.
+
+    A name matches an existing tag only on an exact (case-insensitive) name.
+    The former fallback of taking the first search hit filed "Camping" under
+    the unrelated existing tag "Campingplatz" whenever the exact tag was
+    missing.
+
+    `creatable` holds the casefolded names that cleared the proposal threshold
+    and may be created in WordPress. Everything else is dropped with a log line
+    rather than turned into a new tag - an empty set means this run adds nothing
+    to the taxonomy.
+    """
     ids: list[int] = []
     seen: set[int] = set()
     for tag in tags:
@@ -114,11 +133,11 @@ def _resolve_wp_tag_ids(*, base_url: str, auth_header: str, tags: list[str]) ->
                         tag_id = rid
                         break
             if tag_id is None:
-                    for row in result:
-                        if isinstance(row, dict) and int(row.get("id", 0) or 0) > 0:
-                            tag_id = int(row.get("id", 0))
-                            break
-            if tag_id is None:
+                if name.casefold() not in creatable:
+                    _logger.info(
+                        "Schlagwort '%s' noch nicht etabliert - wird nicht angelegt", name
+                    )
+                    continue
                 created = _wp_request(
                     base_url=base_url,
                     auth_header=auth_header,
@@ -130,6 +149,7 @@ def _resolve_wp_tag_ids(*, base_url: str, auth_header: str, tags: list[str]) ->
                     rid = int(created.get("id", 0) or 0)
                     if rid > 0:
                         tag_id = rid
+                        _logger.info("Schlagwort '%s' in WordPress neu angelegt (#%s)", name, rid)
             if tag_id is not None and tag_id > 0 and tag_id not in seen:
                 seen.add(tag_id)
                 ids.append(tag_id)
@@ -138,6 +158,32 @@ def _resolve_wp_tag_ids(*, base_url: str, auth_header: str, tags: list[str]) ->
     return ids
 
 
+def _creatable_tag_names(tags: list[str], article_id: Any) -> set[str]:
+    """Casefolded names that may be created as new WordPress tags.
+
+    A proposed tag has to show up for `wordpress_new_tag_min_proposals`
+    different articles before it earns a taxonomy entry. Until then it is
+    counted and dropped, so one-off inventions stop accumulating.
+
+    If the bookkeeping itself fails, nothing is creatable: existing tags are
+    still assigned, and the taxonomy simply does not grow that run.
+    """
+    if not tags:
+        return set()
+    settings = get_settings()
+    threshold = max(1, settings.wordpress_new_tag_min_proposals)
+    try:
+        aid = int(article_id) if article_id is not None else None
+    except (TypeError, ValueError):
+        aid = None
+    try:
+        counts = repositories.record_tag_proposals(tags, aid)
+    except Exception as exc:
+        _logger.warning("Schlagwort-Zähler nicht verfügbar, lege keine neuen an: %s", exc)
+        return set()
+    return {name for name, seen in counts.items() if seen >= threshold}
+
+
 _category_id_cache: dict[str, int | None] = {}
 
 
@@ -552,10 +598,14 @@ def publish_article_draft(article: dict[str, Any]) -> tuple[int, str | None]:
 
     wp_post_id = article.get("wp_post_id")
     tag_names = _selected_tags_from_meta(article.get("meta_json"))
+    # Only the leading few tags reach WordPress; the full list still feeds the
+    # category rules below, which were validated against it.
+    wp_tag_names = tag_names[: max(0, settings.wordpress_max_tags_per_post)]
     tag_ids = _resolve_wp_tag_ids(
         base_url=settings.wordpress_base_url,
         auth_header=auth,
-        tags=tag_names,
+        tags=wp_tag_names,
+        creatable=_creatable_tag_names(wp_tag_names, article.get("id")),
     )
     if tag_ids:
         payload["tags"] = tag_ids

backend/tests/test_wordpress.py

@@ -1,9 +1,12 @@
 import os
+import tempfile
 import unittest
+from pathlib import Path
 from unittest.mock import patch
 
 from backend.app import config as config_module
 from backend.app import wordpress as wordpress_module
+from backend.app.db import init_db
 from backend.app.wordpress import publish_article_draft
 
 
@@ -12,15 +15,28 @@ class TestWordpressPublish(unittest.TestCase):
         os.environ["WORDPRESS_BASE_URL"] = "https://example.org"
         os.environ["WORDPRESS_USERNAME"] = "wp-user"
         os.environ["WORDPRESS_APP_PASSWORD"] = "wp-pass"
+        # Publishing records tag proposals, so it needs a database of its own -
+        # otherwise the tests would write into the live one.
+        self.tmp_dir = tempfile.TemporaryDirectory()
+        os.environ["APP_DB_PATH"] = str(Path(self.tmp_dir.name) / "test.db")
         config_module.get_settings.cache_clear()
+        init_db()
         # The category lookup is cached for the process lifetime; without this
         # one test would resolve a slug that the next one expects to be missing.
         wordpress_module._category_id_cache.clear()
 
     def tearDown(self) -> None:
-        for key in ("WORDPRESS_BASE_URL", "WORDPRESS_USERNAME", "WORDPRESS_APP_PASSWORD"):
+        for key in (
+            "WORDPRESS_BASE_URL",
+            "WORDPRESS_USERNAME",
+            "WORDPRESS_APP_PASSWORD",
+            "APP_DB_PATH",
+            "WORDPRESS_MAX_TAGS_PER_POST",
+            "WORDPRESS_NEW_TAG_MIN_PROPOSALS",
+        ):
             os.environ.pop(key, None)
         config_module.get_settings.cache_clear()
+        self.tmp_dir.cleanup()
 
     @patch("backend.app.wordpress._upload_featured_media")
     @patch("backend.app.wordpress._wp_request")
@@ -87,7 +103,7 @@ class TestWordpressPublish(unittest.TestCase):
 
     @patch("backend.app.wordpress._upload_featured_media")
     @patch("backend.app.wordpress._wp_request")
-    def test_publish_resolves_and_sets_tags(self, mock_wp_request, mock_upload_media) -> None:
+    def test_publish_resolves_existing_tags_and_skips_unknown_ones(self, mock_wp_request, mock_upload_media) -> None:
         def _fake_wp_request(**kwargs):
             endpoint = kwargs.get("endpoint", "")
             method = kwargs.get("method", "")
@@ -95,17 +111,13 @@ class TestWordpressPublish(unittest.TestCase):
                 if "Rheingas" in endpoint:
                     return [{"id": 11, "name": "Rheingas"}]
                 return []
-            if method == "POST" and endpoint == "tags":
-                name = (kwargs.get("payload") or {}).get("name")
-                if name == "Gasflasche":
-                    return {"id": 12, "name": "Gasflasche"}
-                return {"id": 13, "name": str(name)}
             if method == "POST" and endpoint == "posts":
                 return {"id": 900, "link": "https://example.org/?p=900"}
             return {}
 
         mock_wp_request.side_effect = _fake_wp_request
         article = {
+            "id": 1,
             "title": "Tag Test",
             "content_raw": "Inhalt",
             "source_url": "https://example.com/source",
@@ -117,7 +129,138 @@ class TestWordpressPublish(unittest.TestCase):
         post_calls = [call for call in mock_wp_request.call_args_list if call.kwargs.get("endpoint") == "posts"]
         self.assertEqual(len(post_calls), 1)
         payload = post_calls[0].kwargs.get("payload", {})
-        self.assertEqual(payload.get("tags"), [11, 12])
+        # "Gasflasche" was proposed for the first time and is not created yet.
+        self.assertEqual(payload.get("tags"), [11])
+        self.assertFalse(
+            any(c.kwargs.get("method") == "POST" and c.kwargs.get("endpoint") == "tags"
+                for c in mock_wp_request.call_args_list)
+        )
+
+    @patch("backend.app.wordpress._upload_featured_media")
+    @patch("backend.app.wordpress._wp_request")
+    def test_new_tag_is_created_once_enough_articles_proposed_it(self, mock_wp_request, mock_upload_media) -> None:
+        created_names: list[str] = []
+
+        def _fake_wp_request(**kwargs):
+            endpoint = kwargs.get("endpoint", "")
+            method = kwargs.get("method", "")
+            if method == "GET" and endpoint.startswith("tags?search="):
+                return []
+            if method == "POST" and endpoint == "tags":
+                created_names.append(str((kwargs.get("payload") or {}).get("name")))
+                return {"id": 55, "name": "Wintercamping"}
+            if method == "POST" and endpoint == "posts":
+                return {"id": 910, "link": "https://example.org/?p=910"}
+            return {}
+
+        mock_wp_request.side_effect = _fake_wp_request
+
+        def _article(article_id: int) -> dict:
+            return {
+                "id": article_id,
+                "title": f"Artikel {article_id}",
+                "content_raw": "Inhalt",
+                "source_url": f"https://example.com/source/{article_id}",
+                "canonical_url": f"https://example.com/source/{article_id}",
+                "meta_json": '{"generated_tags":["Wintercamping"]}',
+            }
+
+        for article_id in (1, 2):
+            publish_article_draft(_article(article_id))
+        self.assertEqual(created_names, [])
+
+        publish_article_draft(_article(3))
+        self.assertEqual(created_names, ["Wintercamping"])
+        payload = [c for c in mock_wp_request.call_args_list if c.kwargs.get("endpoint") == "posts"][-1].kwargs["payload"]
+        self.assertEqual(payload.get("tags"), [55])
+
+    @patch("backend.app.wordpress._upload_featured_media")
+    @patch("backend.app.wordpress._wp_request")
+    def test_republishing_the_same_article_does_not_count_twice(self, mock_wp_request, mock_upload_media) -> None:
+        def _fake_wp_request(**kwargs):
+            endpoint = kwargs.get("endpoint", "")
+            method = kwargs.get("method", "")
+            if method == "GET" and endpoint.startswith("tags?search="):
+                return []
+            if method == "POST" and endpoint == "tags":
+                return {"id": 66, "name": "Solaranlage"}
+            if method == "POST":
+                return {"id": 920, "link": "https://example.org/?p=920"}
+            return {}
+
+        mock_wp_request.side_effect = _fake_wp_request
+        article = {
+            "id": 7,
+            "title": "Immer wieder derselbe Artikel",
+            "content_raw": "Inhalt",
+            "source_url": "https://example.com/source",
+            "canonical_url": "https://example.com/source",
+            "meta_json": '{"generated_tags":["Solaranlage"]}',
+        }
+        for _ in range(4):
+            publish_article_draft(article)
+        self.assertFalse(
+            any(c.kwargs.get("method") == "POST" and c.kwargs.get("endpoint") == "tags"
+                for c in mock_wp_request.call_args_list)
+        )
+
+    @patch("backend.app.wordpress._upload_featured_media")
+    @patch("backend.app.wordpress._wp_request")
+    def test_publish_caps_the_number_of_tags_per_post(self, mock_wp_request, mock_upload_media) -> None:
+        looked_up: list[str] = []
+
+        def _fake_wp_request(**kwargs):
+            endpoint = kwargs.get("endpoint", "")
+            method = kwargs.get("method", "")
+            if method == "GET" and endpoint.startswith("tags?search="):
+                looked_up.append(endpoint)
+                return []
+            if method == "POST" and endpoint == "posts":
+                return {"id": 930, "link": "https://example.org/?p=930"}
+            return {}
+
+        mock_wp_request.side_effect = _fake_wp_request
+        names = [f"Tag{i}" for i in range(1, 13)]
+        article = {
+            "id": 42,
+            "title": "Viele Schlagwörter",
+            "content_raw": "Inhalt",
+            "source_url": "https://example.com/source",
+            "canonical_url": "https://example.com/source",
+            "meta_json": '{"generated_tags":' + str(names).replace("'", '"') + "}",
+        }
+        publish_article_draft(article)
+        self.assertEqual(len(looked_up), 5)
+
+    @patch("backend.app.wordpress._upload_featured_media")
+    @patch("backend.app.wordpress._wp_request")
+    def test_tag_lookup_ignores_near_misses(self, mock_wp_request, mock_upload_media) -> None:
+        """A search hit that is not the exact name must not be reused.
+
+        Plain substring reuse filed "Camping" under the unrelated existing tag
+        "Campingplatz" whenever the exact tag was missing.
+        """
+        def _fake_wp_request(**kwargs):
+            endpoint = kwargs.get("endpoint", "")
+            method = kwargs.get("method", "")
+            if method == "GET" and endpoint.startswith("tags?search="):
+                return [{"id": 99, "name": "Campingplatz"}]
+            if method == "POST" and endpoint == "posts":
+                return {"id": 940, "link": "https://example.org/?p=940"}
+            return {}
+
+        mock_wp_request.side_effect = _fake_wp_request
+        article = {
+            "id": 5,
+            "title": "Naher Treffer",
+            "content_raw": "Inhalt",
+            "source_url": "https://example.com/source",
+            "canonical_url": "https://example.com/source",
+            "meta_json": '{"generated_tags":["Camping"]}',
+        }
+        publish_article_draft(article)
+        payload = [c for c in mock_wp_request.call_args_list if c.kwargs.get("endpoint") == "posts"][0].kwargs["payload"]
+        self.assertNotIn("tags", payload)
 
     @patch("backend.app.wordpress._upload_featured_media")
     @patch("backend.app.wordpress._wp_request")