#!/usr/bin/env python3
"""Reproducible analysis behind /research/when-people-look-up-procrastination/.
Source: Wikimedia REST API, daily pageviews, English Wikipedia, all-access, agent=user (people, not known bots), 2016-01-01 to 2026-09-30.
  python3 scripts/research/wikipedia_pageviews.py            # uses the cached raw file if present
  REFRESH=1 python3 scripts/research/wikipedia_pageviews.py  # refetch everything
Writes: public/research/productivity-pageviews/{daily.csv,summary.csv,day-of-week.svg,months.svg,change.svg}
        src/data/research-pageviews.json (numbers used on the page; nothing on the page is typed by hand)
Method (also on the page):
- Day of week: complete Monday-Sunday weeks from 2023-01-02 to 2026-09-27. For each week, each day's share of that week's views x 7
  (1.00 = an average day). Median across weeks, so one viral day can't move it.
- Month of year: for each complete calendar year 2016-2025, the month's average daily views / that year's average daily views. Median across
  years. Only articles with at least 3 complete years get a month index.
- Long-term change: median daily views in 2025 vs the first complete year (2016 for most), next to English Wikipedia overall
  (aggregate API, same agent filter, average monthly views) over the same years. Median, because single viral days (Flow, Jan 2018: 10x) skew means.
"""
# Run anywhere with Python 3.8+: python3 wikipedia_pageviews.py  (outside the site repo it writes to ./out)
import json, os, csv, urllib.request, urllib.parse, time, statistics as st, datetime as dt
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
if (ROOT / 'src/data').is_dir():  # inside the hourmaxxing-site repo
    OUT = ROOT / 'public/research/productivity-pageviews'; CACHE = ROOT / 'scripts/research/wikipedia_pageviews_raw.json'; JSON_OUT = ROOT / 'src/data/research-pageviews.json'
else:  # downloaded copy: everything goes to ./out
    OUT = Path.cwd() / 'out'; CACHE = OUT / 'raw.json'; JSON_OUT = OUT / 'summary.json'
OUT.mkdir(parents=True, exist_ok=True)
UA = {'User-Agent': 'HourMaxxingResearch/1.0 (https://hourmaxxing.com/contact/; hello@hourmaxxing.com)'}
START, END = '2016010100', '2026093000'
ARTICLES = {'Procrastination': 'Procrastination', 'Motivation': 'Motivation', 'Time_management': 'Time management', 'Pomodoro_Technique': 'Pomodoro Technique',
  'Attention_deficit_hyperactivity_disorder': 'ADHD', 'Habit': 'Habit', 'Getting_Things_Done': 'Getting Things Done', 'Timeboxing': 'Timeboxing',
  'Occupational_burnout': 'Occupational burnout', 'Flow_(psychology)': 'Flow (psychology)', 'Hyperfocus': 'Hyperfocus', 'Executive_dysfunction': 'Executive dysfunction',
  'Decision_fatigue': 'Decision fatigue', 'Planning_fallacy': 'Planning fallacy', 'Doomscrolling': 'Doomscrolling', 'Body_doubling': 'Body doubling',
  'Self-control': 'Self-control', 'Study_skills': 'Study skills', 'Spaced_repetition': 'Spaced repetition', 'Productivity': 'Productivity', 'Boredom': 'Boredom', 'Attention_span': 'Attention span'}
DAYS = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']; MONS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
def get(url): return json.load(urllib.request.urlopen(urllib.request.Request(url, headers=UA), timeout=60))
if CACHE.exists() and not os.environ.get('REFRESH'): raw = json.loads(CACHE.read_text())
else:
    raw = {'articles': {}, 'fetched': dt.date.today().isoformat()}
    for a in ARTICLES:
        d = get(f'https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/en.wikipedia/all-access/user/{urllib.parse.quote(a, safe="")}/daily/{START}/{END}')
        raw['articles'][a] = {i['timestamp'][:8]: i['views'] for i in d['items']}; time.sleep(0.3)
    raw['site_monthly'] = {i['timestamp'][:6]: i['views'] for i in get(f'https://wikimedia.org/api/rest_v1/metrics/pageviews/aggregate/en.wikipedia/all-access/user/monthly/{START}/{END}')['items']}
    CACHE.write_text(json.dumps(raw))
D = lambda s: dt.date(int(s[:4]), int(s[4:6]), int(s[6:8]))
site_year = {}
for k, v in raw['site_monthly'].items(): site_year.setdefault(int(k[:4]), []).append(v)
site_year = {y: sum(v) / 12 for y, v in site_year.items() if len(v) == 12}
rows = []
with open(OUT / 'daily.csv', 'w', newline='') as f:
    w = csv.writer(f); w.writerow(['date', 'article', 'wikipedia_title', 'views'])
    for a, name in ARTICLES.items():
        for d, v in sorted(raw['articles'][a].items()): w.writerow([f'{d[:4]}-{d[4:6]}-{d[6:]}', name, a, v])
for a, name in ARTICLES.items():
    s = {D(d): v for d, v in raw['articles'][a].items()}
    shares = [[] for _ in range(7)]; wk = dt.date(2023, 1, 2)
    while wk + dt.timedelta(6) <= dt.date(2026, 9, 27):
        vals = [s.get(wk + dt.timedelta(i)) for i in range(7)]
        if all(v is not None for v in vals) and sum(vals) > 0:
            t = sum(vals); [shares[i].append(vals[i] / t * 7) for i in range(7)]
        wk += dt.timedelta(7)
    dow = [round(st.median(x), 3) for x in shares]
    mo = [[] for _ in range(12)]; yearly = {}
    for y in range(2016, 2026):
        days = [d for d in s if d.year == y]
        if len(days) < 365: continue
        ya = sum(s[d] for d in days) / len(days); yearly[y] = st.median(s[d] for d in days)  # median day: one viral day can't move it
        for m in range(1, 13):
            md = [s[d] for d in days if d.month == m]; mo[m - 1].append(sum(md) / len(md) / ya)
    moi = [round(st.median(x), 3) for x in mo] if len(mo[0]) >= 3 else None
    first = min(yearly) if yearly else None
    change = round((yearly[2025] / yearly[first] - 1) * 100) if first and 2025 in yearly and first < 2025 else None
    site_change = round((site_year[2025] / site_year[first] - 1) * 100) if change is not None else None
    last12 = [v for d, v in s.items() if dt.date(2025, 10, 1) <= d <= dt.date(2026, 9, 30)]
    rows.append(dict(article=name, title=a, avg_daily_last12=round(sum(last12) / len(last12)), weeks=len(shares[0]), dow=dow, month=moi, years_month=len(mo[0]),
                     first_year=first, median_daily_first_year=round(yearly[first]) if first else None, median_daily_2025=round(yearly[2025]) if 2025 in yearly else None,
                     change_pct=change, wikipedia_change_pct=site_change, url=f'https://en.wikipedia.org/wiki/{a}'))
rows.sort(key=lambda r: -r['avg_daily_last12'])
with open(OUT / 'summary.csv', 'w', newline='') as f:
    w = csv.writer(f)
    w.writerow(['article', 'wikipedia_url', 'avg_daily_views_oct2025_sep2026'] + [f'dow_index_{d.lower()}' for d in DAYS] + [f'month_index_{m.lower()}' for m in MONS] +
               ['first_complete_year', 'median_daily_views_first_year', 'median_daily_views_2025', 'change_pct_first_year_to_2025', 'english_wikipedia_change_pct_same_years'])
    for r in rows: w.writerow([r['article'], r['url'], r['avg_daily_last12']] + r['dow'] + (r['month'] or [''] * 12) + [r['first_year'], r['median_daily_first_year'], r['median_daily_2025'], r['change_pct'], r['wikipedia_change_pct']])
# Headline numbers
sat_low = [r['article'] for r in rows if min(r['dow']) == r['dow'][5]]
peak_day = {}; [peak_day.setdefault(DAYS[r['dow'].index(max(r['dow']))], []).append(r['article']) for r in rows]
with_month = [r for r in rows if r['month']]
peak_month = {}; [peak_month.setdefault(MONS[r['month'].index(max(r['month']))], []).append(r['article']) for r in with_month]
low_month = {}; [low_month.setdefault(MONS[r['month'].index(min(r['month']))], []).append(r['article']) for r in with_month]
med_dow = [round(st.median(r['dow'][i] for r in rows), 3) for i in range(7)]
med_month = [round(st.median(r['month'][i] for r in with_month), 3) for i in range(12)]
summary = dict(range='2016-01-01 to 2026-09-30', dow_range='2023-01-02 to 2026-09-27', fetched=raw.get('fetched'), n=len(rows), sat_low=sat_low, peak_day={k: len(v) for k, v in peak_day.items()},
               peak_month={k: len(v) for k, v in peak_month.items()}, low_month={k: len(v) for k, v in low_month.items()}, n_month=len(with_month), median_dow=med_dow, median_month=med_month,
               site_2016=round(site_year[2016]), site_2025=round(site_year[2025]), site_change=round((site_year[2025] / site_year[2016] - 1) * 100), rows=rows)
JSON_OUT.write_text(json.dumps(summary, indent=1))
# ---------- SVG charts (navy + yellow, house colours). Each carries its own source line so it can be embedded as is.
NAVY, YEL, MUT, LINE, BG2 = '#14213D', '#FFD23F', '#5B6278', '#E2DAC4', '#F3EDDB'
FONT = "font-family=\"Inter, 'Helvetica Neue', Arial, sans-serif\""
SRC = 'Source: HourMaxxing analysis of Wikimedia pageview data. hourmaxxing.com/research/when-people-look-up-procrastination/'
def esc(t): return str(t).replace('&', '&amp;').replace('<', '&lt;')
def mix(c1, c2, t):
    a = [int(c1[i:i + 2], 16) for i in (1, 3, 5)]; b = [int(c2[i:i + 2], 16) for i in (1, 3, 5)]
    return '#' + ''.join(f'{round(x + (y - x) * t):02x}' for x, y in zip(a, b))
# 1. Day-of-week heatmap
srt = sorted(rows, key=lambda r: r['dow'][5])
W, lx, cw, ch, top = 760, 200, 74, 26, 92
H = top + ch * len(srt) + 70
o = [f'<svg xmlns="http://www.w3.org/2000/svg" width="{W}" height="{H}" viewBox="0 0 {W} {H}" role="img" aria-labelledby="t d"><title id="t">Wikipedia views by day of week, 22 productivity topics</title><desc id="d">Index where 1.00 is an average day. Saturday is the lowest day for {len(sat_low)} of {len(rows)} topics.</desc>',
     f'<rect width="{W}" height="{H}" fill="#fff"/>', f'<text x="20" y="34" {FONT} font-size="20" font-weight="700" fill="{NAVY}">Saturday is the quietest day for productivity reading</text>',
     f'<text x="20" y="58" {FONT} font-size="13" fill="{MUT}">English Wikipedia views by day of week, 2023 to 2026. 1.00 = an average day. Darker = more views.</text>']
for i, d in enumerate(DAYS): o.append(f'<text x="{lx + i * cw + cw / 2}" y="{top - 10}" {FONT} font-size="12" font-weight="600" fill="{NAVY}" text-anchor="middle">{d}</text>')
for j, r in enumerate(srt):
    y = top + j * ch
    o.append(f'<text x="{lx - 10}" y="{y + 17}" {FONT} font-size="12.5" fill="{NAVY}" text-anchor="end">{esc(r["article"])}</text>')
    for i, v in enumerate(r['dow']):
        t = max(0, min(1, (v - 0.70) / (1.16 - 0.70)))
        fill = mix('#FFFFFF', NAVY, t * 0.92) if v >= 1 else mix(YEL, '#FFFFFF', max(0, (v - 0.70) / 0.30))
        txt = '#FFFFFF' if (v >= 1 and t > 0.55) else NAVY
        o.append(f'<rect x="{lx + i * cw + 1}" y="{y + 1}" width="{cw - 2}" height="{ch - 2}" fill="{fill}"/><text x="{lx + i * cw + cw / 2}" y="{y + 17}" {FONT} font-size="12" fill="{txt}" text-anchor="middle">{v:.2f}</text>')
o.append(f'<text x="20" y="{H - 34}" {FONT} font-size="12" fill="{MUT}">Yellow = below an average day. Median of up to {max(r["weeks"] for r in srt)} complete weeks per topic (Jan 2023 to Sep 2026).</text>')
o.append(f'<text x="20" y="{H - 14}" {FONT} font-size="11" fill="{MUT}">{esc(SRC)}</text></svg>')
(OUT / 'day-of-week.svg').write_text('\n'.join(o))
# 2. Month of year: median across topics + two highlighted topics
pick = {r['article']: r for r in with_month}
series = [('Median of all topics', med_month, NAVY, 3), ('Pomodoro Technique', pick['Pomodoro Technique']['month'], '#C9A400', 2.2), ('Study skills', pick['Study skills']['month'], MUT, 1.6)]
W, H, L, R, T, B = 760, 430, 60, 30, 90, 90
lo, hi = 0.7, 1.4
X = lambda i: L + i * (W - L - R) / 11; Y = lambda v: T + (hi - v) / (hi - lo) * (H - T - B)
o = [f'<svg xmlns="http://www.w3.org/2000/svg" width="{W}" height="{H}" viewBox="0 0 {W} {H}" role="img" aria-labelledby="t d"><title id="t">Wikipedia views by month, productivity topics</title><desc id="d">Index where 1.00 is an average month. Peaks in January to March, lowest in July.</desc>',
     f'<rect width="{W}" height="{H}" fill="#fff"/>', f'<text x="20" y="34" {FONT} font-size="20" font-weight="700" fill="{NAVY}">Productivity reading peaks in January and dips in July</text>',
     f'<text x="20" y="58" {FONT} font-size="13" fill="{MUT}">English Wikipedia views by month, 2016 to 2025. 1.00 = an average month for that topic.</text>']
for v in (0.8, 1.0, 1.2, 1.4):
    o.append(f'<line x1="{L}" x2="{W - R}" y1="{Y(v)}" y2="{Y(v)}" stroke="{LINE if v != 1.0 else NAVY}" stroke-width="{1 if v != 1.0 else 1}" {"stroke-dasharray=\"3 3\"" if v == 1.0 else ""}/><text x="{L - 8}" y="{Y(v) + 4}" {FONT} font-size="11" fill="{MUT}" text-anchor="end">{v:.1f}</text>')
for i, m in enumerate(MONS): o.append(f'<text x="{X(i)}" y="{H - B + 20}" {FONT} font-size="11.5" fill="{NAVY}" text-anchor="middle">{m}</text>')
ly = H - B + 46
for k, (name, vals, col, sw) in enumerate(series):
    pts = ' '.join(f'{X(i):.1f},{Y(v):.1f}' for i, v in enumerate(vals))
    o.append(f'<polyline points="{pts}" fill="none" stroke="{col}" stroke-width="{sw}" stroke-linejoin="round"/>')
    lx2 = 20 + k * 230; o.append(f'<line x1="{lx2}" x2="{lx2 + 22}" y1="{ly - 4}" y2="{ly - 4}" stroke="{col}" stroke-width="{sw + 0.5}"/><text x="{lx2 + 28}" y="{ly}" {FONT} font-size="12" fill="{NAVY}">{esc(name)}</text>')
o.append(f'<text x="20" y="{H - 14}" {FONT} font-size="11" fill="{MUT}">{esc(SRC)}</text></svg>')
(OUT / 'months.svg').write_text('\n'.join(o))
# 3. Long-term change vs English Wikipedia overall
ch_rows = sorted([r for r in rows if r['change_pct'] is not None], key=lambda r: r['change_pct'])
W, L, bh, T = 760, 200, 22, 96; H = T + bh * (len(ch_rows) + 1) + 70
lo, hi = -100, 150; zx = lambda v: L + (v - lo) / (hi - lo) * (W - L - 40)
o = [f'<svg xmlns="http://www.w3.org/2000/svg" width="{W}" height="{H}" viewBox="0 0 {W} {H}" role="img" aria-labelledby="t d"><title id="t">Change in Wikipedia views since 2016, productivity topics</title><desc id="d">Most general productivity topics fell far more than English Wikipedia overall ({summary["site_change"]}%), while executive dysfunction and hyperfocus grew.</desc>',
     f'<rect width="{W}" height="{H}" fill="#fff"/>', f'<text x="20" y="34" {FONT} font-size="20" font-weight="700" fill="{NAVY}">Fewer people read "Procrastination". More read "Executive dysfunction".</text>',
     f'<text x="20" y="58" {FONT} font-size="13" fill="{MUT}">Change in views on a typical (median) day, first complete year (2016 for most) to 2025. English Wikipedia overall: {summary["site_change"]}%.</text>']
for v in (-100, -50, 0, 50, 100, 150):
    o.append(f'<line x1="{zx(v)}" x2="{zx(v)}" y1="{T - 6}" y2="{T + bh * len(ch_rows)}" stroke="{NAVY if v == 0 else LINE}"/><text x="{zx(v)}" y="{T - 12}" {FONT} font-size="11" fill="{MUT}" text-anchor="middle">{v:+d}%</text>'.replace('+0%', '0%'))
sx = zx(summary['site_change'])
o.append(f'<line x1="{sx}" x2="{sx}" y1="{T - 6}" y2="{T + bh * len(ch_rows)}" stroke="{MUT}" stroke-dasharray="4 3"/>')
for j, r in enumerate(ch_rows):
    y = T + j * bh; v = r['change_pct']; x0, x1 = sorted([zx(0), zx(v)])
    o.append(f'<text x="{L - 10}" y="{y + 15}" {FONT} font-size="12.5" fill="{NAVY}" text-anchor="end">{esc(r["article"])}{" (since " + str(r["first_year"]) + ")" if r["first_year"] != 2016 else ""}</text>')
    o.append(f'<rect x="{x0:.1f}" y="{y + 4}" width="{max(1, x1 - x0):.1f}" height="{bh - 8}" fill="{YEL if v > 0 else NAVY}"/>')
    o.append(f'<text x="{(x1 + 6) if v > 0 else (x0 - 6):.1f}" y="{y + 15}" {FONT} font-size="11.5" fill="{NAVY}" text-anchor="{"start" if v > 0 else "end"}">{v:+d}%</text>' if v > -85 else f'<text x="{x1 + 6:.1f}" y="{y + 15}" {FONT} font-size="11.5" fill="{NAVY}">{v:+d}%</text>')
o.append(f'<text x="20" y="{H - 34}" {FONT} font-size="12" fill="{MUT}">Dashed line = English Wikipedia overall ({summary["site_change"]}%). Page views are not searches; see the methodology.</text>')
o.append(f'<text x="20" y="{H - 14}" {FONT} font-size="11" fill="{MUT}">{esc(SRC)}</text></svg>')
(OUT / 'change.svg').write_text('\n'.join(o))
print(json.dumps({k: v for k, v in summary.items() if k != 'rows'}, indent=1))
for r in ch_rows: print(r['article'], r['change_pct'], r['first_year'])
