forked from bchanot/claude
feat(seo-data): GSC Search Analytics + URL Inspection with lazy OAuth refresh
This commit is contained in:
@@ -0,0 +1,2 @@
|
|||||||
|
{"inspectionResult":{"indexStatusResult":{
|
||||||
|
"verdict":"PASS","coverageState":"Submitted and indexed","lastCrawlTime":"2026-07-01T10:00:00Z"}}}
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
{"rows":[
|
||||||
|
{"keys":["plombier paris"],"clicks":40,"impressions":900,"ctr":0.044,"position":6.3},
|
||||||
|
{"keys":["urgence fuite"],"clicks":5,"impressions":1200,"ctr":0.004,"position":8.9}]}
|
||||||
@@ -1,7 +1,9 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""CrUX + GSC fetch → normalized JSON. Third-party imports are LAZY so mock and
|
"""CrUX + GSC fetch → normalized JSON. Third-party imports are LAZY so mock and
|
||||||
degraded paths run stdlib-only (no venv, no network)."""
|
degraded paths run stdlib-only (no venv, no network)."""
|
||||||
import argparse, json, os
|
import argparse, json, os, sys
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||||
|
|
||||||
def _mock(name):
|
def _mock(name):
|
||||||
d = os.environ.get("SEO_DATA_MOCK_DIR")
|
d = os.environ.get("SEO_DATA_MOCK_DIR")
|
||||||
@@ -61,6 +63,75 @@ def crux(url, strategy="mobile"):
|
|||||||
raw = r.json()
|
raw = r.json()
|
||||||
return _norm_crux(raw)
|
return _norm_crux(raw)
|
||||||
|
|
||||||
|
def _gsc_session(store_path, account):
|
||||||
|
"""Return an authorized requests.Session or a degrade dict. Lazy imports."""
|
||||||
|
rt = None
|
||||||
|
if store_path and account:
|
||||||
|
import tokenstore # local module, stdlib
|
||||||
|
rt = tokenstore.get_refresh_token(store_path, account)
|
||||||
|
cid = os.environ.get("GOOGLE_OAUTH_CLIENT_ID")
|
||||||
|
csec = os.environ.get("GOOGLE_OAUTH_CLIENT_SECRET")
|
||||||
|
if not (rt and cid and csec):
|
||||||
|
return {"status": "degraded", "reason": "no_credentials"}
|
||||||
|
from google.oauth2.credentials import Credentials # lazy
|
||||||
|
from google.auth.transport.requests import AuthorizedSession, Request
|
||||||
|
creds = Credentials(None, refresh_token=rt, client_id=cid, client_secret=csec,
|
||||||
|
token_uri="https://oauth2.googleapis.com/token",
|
||||||
|
scopes=["https://www.googleapis.com/auth/webmasters.readonly"])
|
||||||
|
try:
|
||||||
|
creds.refresh(Request())
|
||||||
|
except Exception as e:
|
||||||
|
# Only a real RefreshError means re-consent; a network blip must NOT
|
||||||
|
# send the user back through OAuth.
|
||||||
|
from google.auth.exceptions import RefreshError # lazy
|
||||||
|
reason = "token_revoked" if isinstance(e, RefreshError) else "network_error"
|
||||||
|
return {"status": "degraded", "reason": reason}
|
||||||
|
return AuthorizedSession(creds)
|
||||||
|
|
||||||
|
def _norm_queries(raw, dim):
|
||||||
|
return {"status": "ok", "source": "gsc", "dimension": dim, "rows": [
|
||||||
|
{"key": r["keys"][0], "clicks": r.get("clicks", 0),
|
||||||
|
"impressions": r.get("impressions", 0), "ctr": r.get("ctr", 0),
|
||||||
|
"position": r.get("position")}
|
||||||
|
for r in raw.get("rows", [])]}
|
||||||
|
|
||||||
|
def queries(store_path, account, property, days=90, dim="query"):
|
||||||
|
raw = _mock("gsc_queries.json")
|
||||||
|
if raw is None:
|
||||||
|
sess = _gsc_session(store_path, account)
|
||||||
|
if isinstance(sess, dict):
|
||||||
|
return sess
|
||||||
|
import datetime as _dt
|
||||||
|
end = _dt.date.today(); start = end - _dt.timedelta(days=days)
|
||||||
|
import urllib.parse
|
||||||
|
url = ("https://searchconsole.googleapis.com/webmasters/v3/sites/"
|
||||||
|
+ urllib.parse.quote(property, safe="") + "/searchAnalytics/query")
|
||||||
|
r = sess.post(url, json={"startDate": start.isoformat(), "endDate": end.isoformat(),
|
||||||
|
"dimensions": [dim], "rowLimit": 100}, timeout=30)
|
||||||
|
if r.status_code == 429:
|
||||||
|
return {"status": "degraded", "reason": "rate_limited"}
|
||||||
|
r.raise_for_status()
|
||||||
|
raw = r.json()
|
||||||
|
return _norm_queries(raw, dim)
|
||||||
|
|
||||||
|
def inspect(store_path, account, property, url):
|
||||||
|
raw = _mock("gsc_inspect.json")
|
||||||
|
if raw is None:
|
||||||
|
sess = _gsc_session(store_path, account)
|
||||||
|
if isinstance(sess, dict):
|
||||||
|
return sess
|
||||||
|
r = sess.post("https://searchconsole.googleapis.com/v1/urlInspection/index:inspect",
|
||||||
|
json={"inspectionUrl": url, "siteUrl": property}, timeout=30)
|
||||||
|
if r.status_code == 429:
|
||||||
|
return {"status": "degraded", "reason": "rate_limited"}
|
||||||
|
r.raise_for_status()
|
||||||
|
raw = r.json()
|
||||||
|
isr = raw["inspectionResult"]["indexStatusResult"]
|
||||||
|
return {"status": "ok", "source": "gsc",
|
||||||
|
"indexed": isr.get("verdict") == "PASS",
|
||||||
|
"coverage": isr.get("coverageState"),
|
||||||
|
"last_crawl": isr.get("lastCrawlTime")}
|
||||||
|
|
||||||
def _cli():
|
def _cli():
|
||||||
p = argparse.ArgumentParser()
|
p = argparse.ArgumentParser()
|
||||||
sub = p.add_subparsers(dest="cmd", required=True)
|
sub = p.add_subparsers(dest="cmd", required=True)
|
||||||
@@ -68,10 +139,27 @@ def _cli():
|
|||||||
pc.add_argument("--url", required=True)
|
pc.add_argument("--url", required=True)
|
||||||
pc.add_argument("--strategy", default="mobile", choices=["mobile", "desktop"])
|
pc.add_argument("--strategy", default="mobile", choices=["mobile", "desktop"])
|
||||||
pc.add_argument("--store", default=None) # accepted+ignored: uniform fetch.sh dispatch
|
pc.add_argument("--store", default=None) # accepted+ignored: uniform fetch.sh dispatch
|
||||||
|
pq = sub.add_parser("queries")
|
||||||
|
pq.add_argument("--store", required=True)
|
||||||
|
pq.add_argument("--account", required=True)
|
||||||
|
pq.add_argument("--property", required=True)
|
||||||
|
pq.add_argument("--days", type=int, default=90)
|
||||||
|
pq.add_argument("--dim", default="query")
|
||||||
|
pi = sub.add_parser("inspect")
|
||||||
|
pi.add_argument("--store", required=True)
|
||||||
|
pi.add_argument("--account", required=True)
|
||||||
|
pi.add_argument("--property", required=True)
|
||||||
|
pi.add_argument("--url", required=True)
|
||||||
args = p.parse_args()
|
args = p.parse_args()
|
||||||
try:
|
try:
|
||||||
if args.cmd == "crux":
|
if args.cmd == "crux":
|
||||||
print(json.dumps(crux(args.url, args.strategy), indent=2))
|
print(json.dumps(crux(args.url, args.strategy), indent=2))
|
||||||
|
elif args.cmd == "queries":
|
||||||
|
print(json.dumps(queries(args.store, args.account, args.property,
|
||||||
|
args.days, args.dim), indent=2))
|
||||||
|
elif args.cmd == "inspect":
|
||||||
|
print(json.dumps(inspect(args.store, args.account, args.property,
|
||||||
|
args.url), indent=2))
|
||||||
except Exception:
|
except Exception:
|
||||||
# Fail-open data contract: ANY unexpected error (HTTP 403/5xx, DNS,
|
# Fail-open data contract: ANY unexpected error (HTTP 403/5xx, DNS,
|
||||||
# timeout) degrades with exit 0 — never a traceback, never empty stdout.
|
# timeout) degrades with exit 0 — never a traceback, never empty stdout.
|
||||||
|
|||||||
@@ -44,6 +44,25 @@ ORIG="$(python3 -c "import sys; sys.path.insert(0,'$SD'); import google_seo; pri
|
|||||||
has "origin strips to host" "$ORIG" 'https://example.com'
|
has "origin strips to host" "$ORIG" 'https://example.com'
|
||||||
hasnt "origin drops the path" "$ORIG" 'blog'
|
hasnt "origin drops the path" "$ORIG" 'blog'
|
||||||
|
|
||||||
|
echo "── gsc (mock) ──"
|
||||||
|
MOCK="$REPO/lib/seo-data/fixtures"
|
||||||
|
TMP2="$(mktemp -d)"; S2="$TMP2/tokens.json"
|
||||||
|
python3 "$SD/tokenstore.py" set --file "$S2" --label client-a --refresh-token RT \
|
||||||
|
--scopes https://www.googleapis.com/auth/webmasters.readonly --properties sc-domain:ex.com >/dev/null
|
||||||
|
Q="$(SEO_DATA_MOCK_DIR="$MOCK" python3 "$SD/google_seo.py" queries \
|
||||||
|
--store "$S2" --account client-a --property sc-domain:ex.com --days 90)"
|
||||||
|
has "queries ok" "$Q" '"status": "ok"'
|
||||||
|
has "queries row key" "$Q" 'plombier paris'
|
||||||
|
has "queries position field" "$Q" '"position": 6.3'
|
||||||
|
I="$(SEO_DATA_MOCK_DIR="$MOCK" python3 "$SD/google_seo.py" inspect \
|
||||||
|
--store "$S2" --account client-a --property sc-domain:ex.com --url https://ex.com/x)"
|
||||||
|
has "inspect indexed true" "$I" '"indexed": true'
|
||||||
|
DEG="$(env -u SEO_DATA_MOCK_DIR python3 "$SD/google_seo.py" queries \
|
||||||
|
--store "$TMP2/none.json" --account nobody --property sc-domain:ex.com)"
|
||||||
|
has "gsc degrades w/o creds" "$DEG" '"status": "degraded"'
|
||||||
|
has "gsc degrade reason" "$DEG" 'no_credentials'
|
||||||
|
rm -rf "$TMP2"
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "seo-data engine: $PASS pass, $FAIL fail"
|
echo "seo-data engine: $PASS pass, $FAIL fail"
|
||||||
[ "$FAIL" -eq 0 ]
|
[ "$FAIL" -eq 0 ]
|
||||||
|
|||||||
Reference in New Issue
Block a user