Get All Comments for a Hacker News Story with the API

import csv
import html
import re
import time
from concurrent.futures import ThreadPoolExecutor

import requests

ALGOLIA = "https://hn.algolia.com/api/v1/items/{}"
FIREBASE = "https://hacker-news.firebaseio.com/v0/item/{}.json"
s = requests.Session()


def get_json(url, attempts=4):
    for i in range(attempts):
        try:
            r = s.get(url, timeout=60)
            if r.status_code == 200:
                return r.json()
            if r.status_code == 404:
                return None
        except requests.RequestException:
            pass
        time.sleep(2 ** i)
    raise RuntimeError(f"failed: {url}")


def to_text(fragment):
    """HN comment HTML -> plain text (<p> becomes a blank line)."""
    t = re.sub(r"<p>", "\n\n", fragment or "")
    t = re.sub(r"<[^>]+>", "", t)
    return html.unescape(t).strip()


def comments_algolia(story_id):
    """One request: the whole tree. Flattened depth-first, in thread order."""
    root = get_json(ALGOLIA.format(story_id))
    rows = []

    def walk(node, depth):
        for c in node.get("children", []):
            rows.append({"id": c["id"], "parent": c["parent_id"], "depth": depth,
                         "author": c["author"], "created_at": c["created_at"],
                         "text": to_text(c["text"])})
            walk(c, depth + 1)

    walk(root, 1)
    return root, rows


def comments_search(story_id, start=0, end=2**31):
    """Flat list from the search index (tags=comment,story_ID). Includes replies
    whose parent was deleted. Splits the time window while a slice has > 1,000 hits."""
    r = s.get("https://hn.algolia.com/api/v1/search_by_date", timeout=60, params={
        "tags": f"comment,story_{story_id}", "hitsPerPage": 1000,
        "numericFilters": f"created_at_i>={start},created_at_i<{end}"}).json()
    if r["nbHits"] <= 1000:
        return [{"id": int(h["objectID"]), "parent": h["parent_id"], "author": h["author"],
                 "created_at": h["created_at"], "text": to_text(h["comment_text"])}
                for h in r["hits"]]
    mid = (start + end) // 2
    return comments_search(story_id, start, mid) + comments_search(story_id, mid, end)


def comments_firebase(story_id, workers=16):
    """Official API: one request per comment, level by level. Keeps deleted
    and dead items so you can see what the Algolia tree leaves out."""
    story = get_json(FIREBASE.format(story_id))
    rows, level, depth = [], [(k, story_id) for k in story.get("kids", [])], 1
    with ThreadPoolExecutor(workers) as ex:
        while level:
            items = list(ex.map(lambda pair: get_json(FIREBASE.format(pair[0])), level))
            nxt = []
            for item in items:
                if not item:
                    continue
                rows.append({"id": item["id"], "parent": item.get("parent"), "depth": depth,
                             "author": item.get("by"), "time": item.get("time"),
                             "deleted": item.get("deleted", False), "dead": item.get("dead", False),
                             "text": to_text(item.get("text"))})
                nxt += [(k, item["id"]) for k in item.get("kids", [])]
            level, depth = nxt, depth + 1
    return story, rows


if __name__ == "__main__":
    story_id = 8863  # "My YC app: Dropbox - Throw away your USB drive"

    t0 = time.time()
    root, rows = comments_algolia(story_id)
    print(f"Algolia: {root['title']!r} -> {len(rows)} comments, "
          f"max depth {max(r['depth'] for r in rows)}, {time.time() - t0:.1f}s")
    print(rows[0]["author"], "|", rows[0]["text"][:70])

    with open(f"hn_{story_id}_comments.csv", "w", newline="", encoding="utf-8") as f:
        w = csv.DictWriter(f, fieldnames=rows[0].keys())
        w.writeheader()
        w.writerows(rows)

    t0 = time.time()
    story, fb_rows = comments_firebase(story_id)
    gone = sum(1 for r in fb_rows if r["deleted"] or r["dead"])
    print(f"Firebase: descendants={story['descendants']} -> {len(fb_rows)} items "
          f"({gone} deleted or dead), {time.time() - t0:.1f}s")

    big = 38309611  # "OpenAI's board has fired Sam Altman"
    t0 = time.time()
    root, rows = comments_algolia(big)
    print(f"Algolia, big thread: {len(rows)} comments, max depth "
          f"{max(r['depth'] for r in rows)}, {time.time() - t0:.1f}s")
    t0 = time.time()
    flat = comments_search(big, start=root["created_at_i"], end=int(time.time()) + 60)
    print(f"Search index, big thread: {len(flat)} comments, {time.time() - t0:.1f}s")
    t0 = time.time()
    story, fb_rows = comments_firebase(big)
    gone = sum(1 for r in fb_rows if r["deleted"] or r["dead"])
    print(f"Firebase, big thread: descendants={story['descendants']} -> {len(fb_rows)} items "
          f"({gone} deleted or dead), {time.time() - t0:.1f}s")

Algolia: 'My YC app: Dropbox - Throw away your USB drive' -> 71 comments, max depth 5, 1.1s
dhouston | oh, and a mac port is coming :)
Firebase: descendants=71 -> 72 items (1 deleted or dead), 5.6s
Algolia, big thread: 2450 comments, max depth 14, 1.8s
Search index, big thread: 2530 comments, 21.8s
Firebase, big thread: descendants=2530 -> 2734 items (204 deleted or dead), 53.9s

id,parent,depth,author,created_at,text
8865,8863,1,dhouston,2007-04-04T19:22:55.000Z,"oh, and a mac port is coming :)"
9007,8865,2,vlad,2007-04-05T01:48:11.000Z,"Drew, this is awesome!  All of the features you mentioned are exactly what people need."