#!/usr/bin/env python3
"""PageHawk engine — read and RANK a folder of documents on a focus query, on your Mac.

Real, on-device document intelligence (stdlib + poppler, no network, nothing uploaded):
  - extract text (pdftotext), render page-1 thumbnails (pdftoppm), page counts (pdfinfo)
  - build a BM25 index; the review UI re-ranks LIVE as you type a focus query (BM25 in JS)
  - near-duplicate grouping by text cosine
  - flag keepers, export a review list (CSV / Markdown)

Usage: pagehawk_engine.py FOLDER [--focus "terms"] [--out out.html]
This is the distinct PageHawk capability layer; Gander Viewer's triage engine is untouched.
"""
import os, sys, re, json, math, subprocess, hashlib, html, argparse, time, glob

STOP = set("""a an the of to in and or for on at by is are was were be been being with as it its this that these those from into over under than then so such not no nor but if while when where which who whom whose will would shall should can could may might must do does did done has have had having i you he she we they them his her our your their my me him us also can per via etc vs""".split())

def run(cmd, timeout=90):
    try:
        r = subprocess.run(cmd, capture_output=True, timeout=timeout)
        return r.returncode, r.stdout, r.stderr
    except Exception as e:
        return 1, b"", str(e).encode()

def extract_text(pdf, limit=120000):
    rc, out, _ = run(["pdftotext", "-q", "-nopgbrk", pdf, "-"])
    t = (out.decode("utf-8", "ignore")) if rc == 0 else ""
    if len(t.strip()) < 120:        # scanned / no text layer -> OCR on-device (shared helper)
        try:
            import sys as _s, os as _o
            _s.path.insert(0, _o.path.dirname(_o.path.abspath(__file__)))
            import pagehawk_diff
            t = pagehawk_diff._ocr_pdf(pdf) or t
        except Exception:
            pass
    return t[:limit]

def page_count(pdf):
    rc, out, _ = run(["pdfinfo", pdf])
    if rc == 0:
        m = re.search(r"Pages:\s+(\d+)", out.decode("utf-8", "ignore"))
        if m: return int(m.group(1))
    return 0

def thumb(pdf, cache):
    key = hashlib.sha1((os.path.abspath(pdf) + str(int(os.path.getmtime(pdf)))).encode()).hexdigest()[:16]
    out = os.path.join(cache, key + ".webp")
    if not os.path.exists(out):
        base = os.path.join(cache, key)
        rc, _, _ = run(["pdftoppm", "-png", "-f", "1", "-l", "1", "-scale-to", "560", pdf, base])
        png = (glob.glob(base + "*.png") or [None])[0]
        if png:
            if subprocess.run(["cwebp", "-quiet", "-q", "82", png, "-o", out]).returncode != 0:
                out = png  # fall back to png if cwebp missing
            else:
                try: os.remove(png)
                except OSError: pass
        else:
            return ""
    return "file://" + os.path.abspath(out)

def toks(text):
    return [w for w in re.findall(r"[a-z][a-z0-9\-]{2,}", text.lower()) if w not in STOP and not w.isdigit()]

def snippet(text, qterms, n=240):
    low = text.lower()
    for q in qterms:
        i = low.find(q)
        if i >= 0:
            s = max(0, i - 90); frag = re.sub(r"\s+", " ", text[s:s + n]).strip()
            return ("…" if s > 0 else "") + frag + "…"
    return re.sub(r"\s+", " ", text[:n]).strip() + "…"

def scan(folder):
    pdfs = []
    for root, _, files in os.walk(folder):
        if any(p.startswith(".") for p in root.split(os.sep)): continue
        for f in sorted(files):
            if f.lower().endswith(".pdf"): pdfs.append(os.path.join(root, f))
    return pdfs

def build(folder, focus, out_path):
    cache = os.path.join(os.path.dirname(os.path.abspath(out_path)) or ".", "cache")
    os.makedirs(cache, exist_ok=True)
    pdfs = scan(folder)
    if not pdfs:
        print("No PDFs found in", folder); return None
    docs, df = [], {}
    for p in pdfs:
        text = extract_text(p)
        tf = {}
        for w in toks(text): tf[w] = tf.get(w, 0) + 1
        for w in tf: df[w] = df.get(w, 0) + 1
        docs.append({
            "id": len(docs), "name": os.path.basename(p), "path": p,
            "pages": page_count(p), "mtime": int(os.path.getmtime(p)),
            "thumb": thumb(p, cache), "len": sum(tf.values()), "tf": tf,
            "text": text,
        })
        print("  indexed", os.path.basename(p), "(", len(text), "chars )")
    N = len(docs); avgdl = sum(d["len"] for d in docs) / max(1, N)

    # near-duplicate grouping (text cosine on tf vectors)
    def cos(a, b):
        common = set(a) & set(b)
        if not common: return 0.0
        dot = sum(a[t] * b[t] for t in common)
        na = math.sqrt(sum(v * v for v in a.values())); nb = math.sqrt(sum(v * v for v in b.values()))
        return dot / (na * nb) if na and nb else 0.0
    parent = list(range(N))
    def find(x):
        while parent[x] != x: parent[x] = parent[parent[x]]; x = parent[x]
        return x
    for i in range(N):
        for j in range(i + 1, N):
            if cos(docs[i]["tf"], docs[j]["tf"]) >= 0.82:
                parent[find(i)] = find(j)
    groups = {}
    for i in range(N): groups.setdefault(find(i), []).append(i)
    dup_groups = [g for g in groups.values() if len(g) > 1]

    # bound the embedded index: top 280 terms per doc by tf
    used = set()
    for d in docs:
        d["top"] = dict(sorted(d["tf"].items(), key=lambda kv: -kv[1])[:280])
        used.update(d["top"].keys())
    DF = {t: df[t] for t in used}
    DOCS = [{"id": d["id"], "name": d["name"], "pages": d["pages"], "mtime": d["mtime"],
             "thumb": d["thumb"], "len": d["len"], "tf": d["top"],
             "snippet": snippet(d["text"], toks(focus))} for d in docs]
    IDX = {"N": N, "avgdl": round(avgdl, 2), "df": DF}
    payload = {"docs": DOCS, "idx": IDX, "focus": focus,
               "folder": os.path.abspath(folder),
               "dupGroups": dup_groups, "built": time.strftime("%Y-%m-%d %H:%M")}
    htmlout = TEMPLATE.replace("/*DATA*/", json.dumps(payload))
    with open(out_path, "w", encoding="utf-8") as f:
        f.write(htmlout)
    print(f"\nPageHawk: ranked {N} documents on focus '{focus}'. {len(dup_groups)} near-duplicate group(s).")
    print("wrote", out_path)
    return out_path


TEMPLATE = r"""<!doctype html><html lang="en" data-theme="light"><head><meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<title>PageHawk</title>
<script>document.documentElement.dataset.theme=new URLSearchParams(location.search).get('theme')||'light';</script>
<style>
/* LIGHT is the default (Jason 2026-07-04, app-wide policy); the Swift shell pushes the user's
 * actual appearance (dataset.theme) on load; ?theme=dark works standalone. */
:root{--ink:#FBFAF7;--ink2:#F4F2EB;--ink3:#FFFFFF;--s2:#F0EDE4;--line:#E3DFD3;
 --gold:#C8A872;--goldlt:#8A6516;--text:#15171A;--muted:#6A6A6B;--dim:#9A958C;
 --on-gold:#1a1a1a;--tbar:linear-gradient(#FFFFFF,#F6F3EC);--panel:#F4F2EB;
 --sbtxt:#5A554A;--dot:#D8D2C4;--nm:#3A362E;--pillbg:rgba(255,255,255,.9);--pilltxt:#15171A;
 --serif:'Cambria','Big Caslon','Hoefler Text',Georgia,serif;--sans:-apple-system,'Inter',system-ui,sans-serif}
html[data-theme="dark"]{--ink:#15171A;--ink2:#1A1D21;--ink3:#1D2024;--s2:#23272C;--line:#2A2D31;
 --goldlt:#E2C998;--text:#F6F1E7;--muted:#9A917E;--dim:#6E6757;
 --on-gold:#15171A;--tbar:linear-gradient(#212529,#1A1D21);--panel:#181B1E;
 --sbtxt:#B8B0A0;--dot:#3A3E44;--nm:#C9C1B1;--pillbg:rgba(21,23,26,.82);--pilltxt:#C9C1B1}
*{margin:0;padding:0;box-sizing:border-box}
body{background:var(--ink);color:var(--text);font-family:var(--sans);height:100vh;overflow:hidden;font-size:13px}
.app{display:flex;flex-direction:column;height:100vh}
.titlebar{height:40px;display:flex;align-items:center;justify-content:center;border-bottom:.5px solid var(--line);
 background:var(--tbar);font-size:12.5px;color:var(--muted);font-weight:500}
.titlebar .wm{font-family:var(--serif);color:var(--text);font-size:14px;margin-right:7px}
.toolbar{display:flex;align-items:center;gap:12px;padding:9px 16px;border-bottom:.5px solid var(--line);background:var(--ink2)}
.coll{display:flex;align-items:center;gap:9px;font-size:14px;font-weight:500;color:var(--text)}
.coll .ic{width:26px;height:26px;border-radius:7px;background:var(--s2);border:.5px solid var(--line);display:flex;align-items:center;justify-content:center;color:var(--gold)}
.seg{display:flex;background:var(--s2);border:.5px solid var(--line);border-radius:9px;padding:3px;gap:2px}
.seg button{font-family:var(--sans);font-size:12.5px;color:var(--muted);padding:5px 13px;border:none;border-radius:6px;background:none;cursor:pointer}
.seg button.on{background:var(--gold);color:var(--on-gold);font-weight:600}
.focuswrap{display:flex;align-items:center;gap:8px;flex:1;max-width:520px;background:var(--s2);border:.5px solid rgba(200,168,114,.42);border-radius:8px;padding:0 12px;height:34px}
.focuswrap .fl{font-size:9.5px;letter-spacing:.12em;text-transform:uppercase;color:var(--muted);flex:none}
.focuswrap input{flex:1;background:none;border:none;outline:none;color:var(--goldlt);font-family:var(--sans);font-size:13px}
.focuswrap input::placeholder{color:var(--dim)}
.tb-sp{flex:1}
.btn{font-size:12.5px;border-radius:8px;padding:8px 15px;border:.5px solid var(--line);background:var(--s2);color:var(--text);cursor:pointer}
.btn.gold{background:var(--gold);color:var(--on-gold);font-weight:600;border-color:var(--gold)}
.body{display:flex;flex:1;min-height:0}
.sidebar{width:204px;flex:none;border-right:.5px solid var(--line);background:var(--panel);padding:14px 11px;overflow:auto}
.sb-h{font-size:10px;letter-spacing:.15em;text-transform:uppercase;color:var(--dim);margin:6px 8px 8px;font-weight:600}
.sb-h:not(:first-child){margin-top:18px}
.sb-i{display:flex;align-items:center;gap:9px;padding:7px 9px;border-radius:7px;font-size:13px;color:var(--sbtxt);cursor:pointer}
.sb-i.on{background:var(--s2);color:var(--text)}.sb-i .d{width:7px;height:7px;border-radius:2px;background:var(--dot)}
.sb-i.on .d{background:var(--gold)}.sb-i .ct{margin-left:auto;font-size:11px;color:var(--dim)}
.canvas{flex:1;padding:18px;overflow:auto;background:var(--ink)}
.cv-h{display:flex;align-items:baseline;gap:10px;margin-bottom:14px}.cv-h b{font-size:15px}.cv-h span{font-size:12px;color:var(--dim)}
.grid{display:grid;grid-template-columns:repeat(auto-fill,minmax(150px,1fr));gap:15px}
.card{position:relative;background:var(--ink3);border:.5px solid var(--line);border-radius:10px;overflow:hidden;cursor:pointer;transition:transform .12s,border-color .12s}
.card:hover{transform:translateY(-2px)}.card.sel{border-color:var(--gold);box-shadow:0 0 0 2px rgba(200,168,114,.28)}
.card.flagged{border-color:var(--gold)}
.thumb{aspect-ratio:3/4;background:#fff;overflow:hidden}.thumb img{width:100%;height:100%;object-fit:cover;object-position:top center}
.meta{padding:8px 10px}.nm{font-size:11px;color:var(--nm);white-space:nowrap;overflow:hidden;text-overflow:ellipsis;font-weight:500}
.sub{font-size:10px;color:var(--dim);margin-top:2px}
.pill{position:absolute;top:8px;left:8px;font-size:11px;font-weight:600;background:var(--pillbg);color:var(--pilltxt);border-radius:6px;padding:2px 7px}
/* The near-duplicate view has no focus score to show, so its pill renders empty. On a
 * light ground that is invisible; on a dark one it is a blank chip on every card. */
.pill:empty{display:none}
.card.top .pill{background:var(--gold);color:var(--on-gold)}
.flag{position:absolute;top:8px;right:8px;font-size:8.5px;letter-spacing:.1em;font-weight:700;color:var(--on-gold);background:var(--gold);border-radius:4px;padding:2px 6px;display:none}
.card.flagged .flag{display:block}
.inspector{width:268px;flex:none;border-left:.5px solid var(--line);background:var(--panel);padding:16px;overflow:auto}
.in-thumb{width:118px;aspect-ratio:3/4;border-radius:8px;overflow:hidden;border:.5px solid var(--line);background:#fff;margin-bottom:13px}
.in-thumb img{width:100%;height:100%;object-fit:cover;object-position:top center}
.in-t{font-size:13.5px;font-weight:600;line-height:1.3}.in-s{font-size:11.5px;color:var(--muted);margin-top:3px}
.in-sec{font-size:10px;letter-spacing:.13em;text-transform:uppercase;color:var(--dim);margin:16px 0 9px;font-weight:600}
.scorebig{font-family:var(--serif);font-size:34px;color:var(--gold)}
.term{display:inline-flex;align-items:center;gap:5px;font-size:11.5px;color:var(--nm);background:var(--s2);border:.5px solid var(--line);border-radius:6px;padding:3px 8px;margin:0 5px 5px 0}
.term b{color:var(--gold)}
.snip{font-size:11.5px;color:var(--muted);line-height:1.5;background:var(--ink);border:.5px solid var(--line);border-radius:7px;padding:10px}
.kbd{margin-top:14px;display:flex;gap:6px;flex-wrap:wrap}.kbd span{font-size:10.5px;color:var(--muted);background:var(--s2);border:.5px solid var(--line);border-radius:5px;padding:3px 7px}
.statusbar{height:32px;display:flex;align-items:center;gap:8px;padding:0 16px;border-top:.5px solid var(--line);background:var(--ink2);font-size:11.5px;color:var(--dim)}
.statusbar .lock{color:var(--gold)}.statusbar .sp{flex:1}
.toast{position:fixed;bottom:46px;left:50%;transform:translateX(-50%);background:var(--gold);color:var(--on-gold);font-weight:600;font-size:12.5px;padding:9px 18px;border-radius:9px;opacity:0;transition:opacity .2s;pointer-events:none}
.toast.show{opacity:1}
.empty{color:var(--dim);font-size:13px;padding:30px}
</style></head><body>
<div class="app">
  <div class="titlebar"><span class="wm">PageHawk</span></div>
  <div class="toolbar">
    <div class="coll"><span class="ic">&#9638;</span><span id="collName">Collection</span></div>
    <div class="seg"><button class="on" data-view="rank" id="vRank">Ranked</button><button data-view="groups" id="vGroups">Near-duplicates</button></div>
    <div class="focuswrap"><span class="fl">Focus</span><input id="focus" placeholder="type what matters: a clause, a term, a topic"></div>
    <div class="tb-sp"></div>
    <button class="btn" id="flagAll">Flag top 5</button>
    <button class="btn gold" id="export">Export review list</button>
  </div>
  <div class="body">
    <aside class="sidebar" id="sidebar"></aside>
    <main class="canvas" id="canvas"></main>
    <aside class="inspector" id="inspector"></aside>
  </div>
  <div class="statusbar"><span class="lock">&#128274;</span> <span id="status">Read and ranked on your Mac. Nothing uploaded.</span><span class="sp"></span><span id="count"></span></div>
</div>
<div class="toast" id="toast"></div>
<script>
const DATA = /*DATA*/;
const k1=1.5, b=0.75, IDX=DATA.idx, DOCS=DATA.docs;
const flagged=new Set(); let sel=0, view='rank';
function qterms(s){return (s||'').toLowerCase().match(/[a-z][a-z0-9\-]{2,}/g)||[];}
function bm25(doc,terms){let s=0;for(const t of terms){const tf=doc.tf[t]||0;if(!tf)continue;const df=IDX.df[t]||0;
  const idf=Math.log(1+(IDX.N-df+0.5)/(df+0.5));s+=idf*(tf*(k1+1))/(tf+k1*(1-b+b*doc.len/IDX.avgdl));}return s;}
function ranked(){const terms=qterms(document.getElementById('focus').value);
  let scored=DOCS.map(d=>({d,raw:terms.length?bm25(d,terms):d.mtime}));
  const max=Math.max(...scored.map(x=>x.raw),0.0001);
  scored.forEach(x=>{x.score=terms.length? (x.raw>0?Math.round(28+71*x.raw/max):0) : Math.round(40+59*x.raw/max);});
  scored.sort((a,b)=>b.score-a.score);return {scored,terms};}
function whyTerms(doc,terms){return terms.map(t=>({t,n:doc.tf[t]||0})).filter(x=>x.n>0).sort((a,b)=>b.n-a.n);}
function card(x,i,terms){const d=x.d;const top=i===0&&x.score>0?' top':'';const fl=flagged.has(d.id)?' flagged':'';const s=d.id===DOCS[sel]?.id?' sel':'';
  return `<div class="card${top}${fl}${s}" data-id="${d.id}"><span class="pill">${x.score}</span><span class="flag">FLAG</span>
   <div class="thumb">${d.thumb?`<img src="${d.thumb}" loading="lazy">`:''}</div>
   <div class="meta"><div class="nm">${esc(d.name)}</div><div class="sub">${d.pages?d.pages+' pp':''}</div></div></div>`;}
function esc(s){return (s||'').replace(/[&<>"]/g,c=>({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;'}[c]));}
function render(){const {scored,terms}=ranked();const cv=document.getElementById('canvas');
  document.getElementById('collName').textContent=basename(DATA.folder);
  document.getElementById('count').textContent=`${DOCS.length} documents · ${flagged.size} flagged`;
  if(view==='rank'){
    cv.innerHTML=`<div class="cv-h"><b>${terms.length?'Ranked on your focus':'Most recent first'}</b><span>${terms.length?'scored on '+esc(terms.join(', '))+' · on your Mac':'type a focus query to rank'}</span></div>
      <div class="grid">${scored.map((x,i)=>card(x,i,terms)).join('')}</div>`;
  } else {
    const groups=DATA.dupGroups;
    cv.innerHTML=`<div class="cv-h"><b>Near-duplicates</b><span>${groups.length} group(s) found by text similarity · ${DOCS.length} documents</span></div>`+
      (groups.length? groups.map(g=>`<div style="margin-bottom:14px"><div style="font-size:12px;color:var(--muted);margin-bottom:8px">${g.length} near-duplicates</div><div class="grid">${g.map((id,i)=>card({d:DOCS[id],score:''},i,terms)).join('')}</div></div>`).join('')
        : `<div class="empty">No near-duplicates in this set — every document is distinct. (PageHawk groups by text similarity at 0.82 cosine.)</div>`);
  }
  cv.querySelectorAll('.card').forEach(c=>{c.onclick=()=>{sel=DOCS.findIndex(d=>d.id==c.dataset.id);renderInspector(terms);highlightSel();};});
  renderSidebar(scored);renderInspector(terms);
}
function highlightSel(){document.querySelectorAll('.card').forEach(c=>c.classList.toggle('sel',c.dataset.id==DOCS[sel].id));}
function basename(p){return (p||'').split('/').filter(Boolean).pop()||'Collection';}
function renderSidebar(scored){const top=scored[0];const sb=document.getElementById('sidebar');
  sb.innerHTML=`<div class="sb-h">Collection</div>
   <div class="sb-i on"><span class="d"></span>All documents<span class="ct">${DOCS.length}</span></div>
   <div class="sb-i"><span class="d"></span>Flagged<span class="ct">${flagged.size}</span></div>
   <div class="sb-i"><span class="d"></span>Near-duplicates<span class="ct">${DATA.dupGroups.reduce((a,g)=>a+g.length,0)}</span></div>
   <div class="sb-h">Top match</div>
   <div style="font-size:12px;color:var(--muted);padding:0 9px;line-height:1.5">${top&&top.score?esc(top.d.name)+' · '+top.score:'—'}</div>`;}
function renderInspector(terms){const d=DOCS[sel];if(!d){document.getElementById('inspector').innerHTML='';return;}
  const {scored}=ranked();const x=scored.find(s=>s.d.id===d.id)||{score:0};const why=whyTerms(d,terms);
  document.getElementById('inspector').innerHTML=`
   <div class="in-thumb">${d.thumb?`<img src="${d.thumb}">`:''}</div>
   <div class="in-t">${esc(d.name)}</div><div class="in-s">${d.pages?d.pages+' pages':''} · PDF${flagged.has(d.id)?' · <span style="color:var(--gold)">flagged</span>':''}</div>
   <div class="in-sec">Focus score</div><div class="scorebig">${x.score}</div>
   <div class="in-sec">Why it ranks</div>
   ${why.length? why.map(w=>`<span class="term"><b>${esc(w.t)}</b> ×${w.n}</span>`).join('') : '<div style="font-size:11.5px;color:var(--dim)">No focus terms in this document. Ranked by recency.</div>'}
   <div class="in-sec">In context</div><div class="snip">${esc(d.snippet)}</div>
   <div class="kbd"><span>F flag</span><span>E export</span><span>type to re-rank</span></div>`;}
function toast(m){const t=document.getElementById('toast');t.textContent=m;t.classList.add('show');setTimeout(()=>t.classList.remove('show'),1600);}
function doExport(){const rows=[...flagged].map(id=>DOCS.find(d=>d.id===id)).filter(Boolean);
  if(!rows.length){toast('Flag some documents first');return;}
  const {scored}=ranked();const sc=id=>{const x=scored.find(s=>s.d.id===id);return x?x.score:'';};
  let md=`# PageHawk review list\\nFocus: ${document.getElementById('focus').value||'(recency)'}\\n${rows.length} documents · ${DATA.built}\\n\\n`;
  rows.forEach(d=>md+=`- [${sc(d.id)}] ${d.name} (${d.pages} pp)\\n`);
  const blob=new Blob([md],{type:'text/markdown'});const a=document.createElement('a');a.href=URL.createObjectURL(blob);a.download='pagehawk-review-list.md';a.click();
  toast('Exported '+rows.length+' documents');}
document.getElementById('focus').addEventListener('input',render);
document.getElementById('export').onclick=doExport;
document.getElementById('flagAll').onclick=()=>{const {scored}=ranked();scored.slice(0,5).forEach(x=>flagged.add(x.d.id));render();toast('Flagged top 5');};
document.querySelectorAll('.seg button').forEach(btn=>btn.onclick=()=>{view=btn.dataset.view;document.querySelectorAll('.seg button').forEach(b=>b.classList.toggle('on',b===btn));render();});
document.addEventListener('keydown',e=>{if(e.target.tagName==='INPUT')return;
  if(e.key==='f'||e.key==='F'){const id=DOCS[sel].id;flagged.has(id)?flagged.delete(id):flagged.add(id);render();}
  if(e.key==='e'||e.key==='E')doExport();
  if(e.key==='ArrowRight'){sel=Math.min(DOCS.length-1,sel+1);render();}
  if(e.key==='ArrowLeft'){sel=Math.max(0,sel-1);render();}});
document.getElementById('focus').value=DATA.focus||'';
{const r0=ranked();const top=r0.scored[0];if(top){const i=DOCS.findIndex(d=>d.id===top.d.id);if(i>=0)sel=i;}}
render();
</script></body></html>"""


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("folder")
    ap.add_argument("--focus", default="")
    ap.add_argument("--out", default=os.path.expanduser("~/Developer/pagehawk/review.html"))
    a = ap.parse_args()
    build(a.folder, a.focus, a.out)


if __name__ == "__main__":
    main()
