feat(seo-data): C2 — cannibalisation from Google's own data, one param away

The inventory called this "no duplicate-content / cannibalisation detection".
Splitting that into its two halves shows one is free and the other is a trap.

CANNIBALISATION — free, and the data was already reachable. Search Analytics
has always accepted several dimensions at once ("no limit to the number of
dimensions that you can group by"); this engine only ever sent
`"dimensions": [dim]` and _norm_queries only ever read `keys[0]`. So
query+page — the pairing that exposes the conflict — was one parameter away
and nobody asked. Same shape of win as W1.

  fetch.sh cannibal → queries where 2+ of OUR pages compete, ranked by total
  impressions, strongest page first inside each. Same auth, same quota family,
  no new scope. `capped` reports a full row window rather than presenting a
  truncated list as exhaustive — same rule as COVERAGE and the sitemap caps.

Grouping happens in the engine, deterministically: asking an LLM to group
1000 rows by query is arithmetic it should never be handed.

Backward compatible: rows gained `keys` (the list the API actually returns);
`key` stays as keys[0], so the single-dim quick-wins consumer is untouched.
A test pins both.

30/70 DUPLICATION — deliberately NOT built, and this is the honest half.
Measuring it needs main-content extraction (strip nav/header/footer). Without
that, comparing two same-template pages returns ~95% similar for every site —
a confident false positive, which is exactly the failure class the rest of
this branch exists to remove. It stays an explicit LLM judgement over the >=3
same-family pages C1c now samples for it, labelled as judgement, never quoting
a similarity percentage nobody computed. A wrong number would be worse than
the current honest gap.

The two must not be merged in the report either: cannibalisation is a SERP
fact Google measured; 30/70 is a content question. The spec now says so.

Verified: fixture with 3 pages on one query, 2 on another, 1 on a third →
2 conflicts, correct ranking, single-page query excluded; live dispatch
degrades cleanly with no account; seo-data 110 -> 119 pass, 0 fail; full suite
green; shellcheck + py_compile clean.
This commit is contained in:
Bastien Chanot
2026-07-17 11:55:00 +02:00
parent 04ccc5ad9b
commit 3a15643c2c
6 changed files with 139 additions and 7 deletions
+61 -5
View File
@@ -89,13 +89,16 @@ def _gsc_session(store_path, account):
return AuthorizedSession(creds)
def _norm_queries(raw, dim):
# `keys` is the list the API actually returns (one entry per requested
# dimension); `key` stays as keys[0] so the single-dim consumer that reads
# it keeps working. Additive — nothing to migrate.
return {"status": "ok", "source": "gsc", "dimension": dim, "rows": [
{"key": r["keys"][0], "clicks": r.get("clicks", 0),
{"key": r["keys"][0], "keys": r["keys"], "clicks": r.get("clicks", 0),
"impressions": r.get("impressions", 0), "ctr": r.get("ctr", 0),
"position": r.get("position")}
for r in raw.get("rows", [])]}
def queries(store_path, account, property, days=90, dim="query"):
def queries(store_path, account, property, days=90, dim="query", rows=100):
raw = _mock("gsc_queries.json")
if raw is None:
sess = _gsc_session(store_path, account)
@@ -106,8 +109,12 @@ def queries(store_path, account, property, days=90, dim="query"):
import urllib.parse
url = ("https://searchconsole.googleapis.com/webmasters/v3/sites/"
+ urllib.parse.quote(property, safe="") + "/searchAnalytics/query")
# dim accepts a comma-separated list: the API groups by several
# dimensions at once ("no limit... but you cannot group by the same
# dimension twice"), and query+page is what exposes cannibalisation.
dims = [d.strip() for d in dim.split(",") if d.strip()]
r = sess.post(url, json={"startDate": start.isoformat(), "endDate": end.isoformat(),
"dimensions": [dim], "rowLimit": 100}, timeout=30)
"dimensions": dims, "rowLimit": rows}, timeout=30)
if r.status_code == 429:
return {"status": "degraded", "reason": "rate_limited"}
r.raise_for_status()
@@ -147,6 +154,44 @@ def _norm_rich(ir):
"errors": errors, "warnings": warnings, "issues": msgs})
return {"verdict": rr.get("verdict"), "types": types}
def _group_by_query(rows):
"""query+page rows -> {query: [row, …]}. Deterministic aggregation, not
judgement: the agent must not be asked to group 1000 rows by eye."""
by_q = {}
for r in rows:
keys = r.get("keys") or []
if len(keys) < 2:
continue
by_q.setdefault(keys[0], []).append(
{"url": keys[1], "clicks": r["clicks"],
"impressions": r["impressions"], "position": r["position"]})
return by_q
def cannibal(store_path, account, property, days=90, rows=1000):
"""Queries where 2+ of our own pages compete for the same term.
Google's own data says it; nothing in this system asked. Cannibalisation
is a SERP fact, not a content-similarity guess — do not confuse it with
the 30/70 duplication rule, which has no data source here."""
res = queries(store_path, account, property, days, "query,page", rows)
if res.get("status") != "ok":
return res
conflicts = []
for q, pages in _group_by_query(res["rows"]).items():
if len(pages) < 2:
continue
pages.sort(key=lambda p: p["impressions"], reverse=True)
conflicts.append({"query": q, "pages": len(pages),
"total_impressions": sum(p["impressions"] for p in pages),
"urls": pages})
conflicts.sort(key=lambda c: c["total_impressions"], reverse=True)
return {"status": "ok", "source": "gsc", "days": days,
"rows_scanned": len(res["rows"]),
# rows_scanned == rows means the window was FULL: there may be more
# conflicts past the cut. Reported, never silently truncated.
"capped": len(res["rows"]) >= rows,
"conflict_count": len(conflicts), "conflicts": conflicts}
def inspect(store_path, account, property, url):
raw = _mock("gsc_inspect.json")
if raw is None:
@@ -182,7 +227,15 @@ def _cli():
pq.add_argument("--account", required=True)
pq.add_argument("--property", required=True)
pq.add_argument("--days", type=int, default=90)
pq.add_argument("--dim", default="query")
pq.add_argument("--dim", default="query",
help="one dimension, or a comma-separated list (query,page)")
pq.add_argument("--rows", type=int, default=100)
pn = sub.add_parser("cannibal")
pn.add_argument("--store", required=True)
pn.add_argument("--account", required=True)
pn.add_argument("--property", required=True)
pn.add_argument("--days", type=int, default=90)
pn.add_argument("--rows", type=int, default=1000)
pi = sub.add_parser("inspect")
pi.add_argument("--store", required=True)
pi.add_argument("--account", required=True)
@@ -193,7 +246,10 @@ def _cli():
print(json.dumps(crux(args.url, args.strategy), indent=2))
elif args.cmd == "queries":
print(json.dumps(queries(args.store, args.account, args.property,
args.days, args.dim), indent=2))
args.days, args.dim, args.rows), indent=2))
elif args.cmd == "cannibal":
print(json.dumps(cannibal(args.store, args.account, args.property,
args.days, args.rows), indent=2))
elif args.cmd == "inspect":
print(json.dumps(inspect(args.store, args.account, args.property,
args.url), indent=2))