How do I keep Crawlbase's site tiers from surprising my bill?

// npm i crawlbase   (Node 18+)   env: CRAWLBASE_TOKEN, UPROCK_API_KEYimport { CrawlingAPI } from "crawlbase";​// Crawlbase asks for a client timeout of at least 90 s.const crawlbase = new CrawlingAPI({  token: process.env.CRAWLBASE_TOKEN ?? "",  timeout: 90_000,});​// Crawlbase reports each domain's price tier on billed responses (X-Domain-Complexity).// A domain billed at any other tier goes straight to UpRock from then on.const ACCEPTED_TIERS = new Set(["standard"]);const uprockDomains = new Set<string>(); // save it somewhere if you want it to persist​// HTTP statuses worth a second network: a bad token (401), your spend cap or account// limit (402, 403), the concurrency limit (429), errors and timeouts (500, 503, 504).const TRY_UPROCK_HTTP = new Set([401, 402, 403, 429, 500, 503, 504]);​// cb_status (crawlbase.com/docs/status-codes, checked 25 Oct 2026): 704 is the site's own// error, worth a second try only if it's a 5xx; 595 is a click selector to fix; 503 (a// block page), 705 (the site said 403), 706 (429) and other 5xx to 7xx codes are worth it.function worthUpRock(http: number, cb: number, site: number) {  if (cb === 704) return site >= 500;  if (cb === 595) return false;  return cb >= 500 || TRY_UPROCK_HTTP.has(http);}​async function viaUpRock(url: string, country: string) {  // One call: UpRock loads the page on a real device in that country, tries a fresh device  // every 7 s (up to 3 tries) and returns markdown. Failed calls aren't billed.  const res = await fetch("https://edge.uprock.com/v1/crawl", {    method: "POST",    headers: {      Authorization: `Bearer ${process.env.UPROCK_API_KEY}`,      "Content-Type": "application/json",    },    body: JSON.stringify({ url, country }),    signal: AbortSignal.timeout(90_000),  });  const page = await res.json();  if (!res.ok) throw new Error(`UpRock ${res.status} ${page.code}: ${page.error}`);  return page.markdown ?? "";}​async function viaCrawlbase(url: string, country: string) {  try {    const res = await crawlbase.get(url, { format: "md", country });    const header = (name: string) => String(res.headers[name] ?? "");    return {      http: Number(res.statusCode),      cb: Number(header("cb_status") || header("pc_status")), // pc_status: the old name      site: Number(header("original_status")),      tier: header("x-domain-complexity") || header("domain_complexity"),      body: String(res.body),    };  } catch (err) {    console.warn(`Crawlbase unreachable or timed out (${err}); trying UpRock`);    return null;  }}​export async function read(url: string, country = "US") {  const domain = new URL(url).hostname;  const r = uprockDomains.has(domain) ? null : await viaCrawlbase(url, country);  if (r) {    if (r.tier && !ACCEPTED_TIERS.has(r.tier)) {      uprockDomains.add(domain); // this page is billed already; the next ones aren't      console.warn(        `Crawlbase bills ${domain} as "${r.tier}"; using UpRock from now on`,      );    }    if (r.http === 200 && r.cb === 200) {      // 404 and 410 are the site's answer (and billed): a second crawler will see the same.      if (r.site === 404 || r.site === 410)        throw new Error(`The page is gone (${r.site})`);      return { via: "crawlbase", markdown: r.body };    }    if (r.cb !== 207 && !worthUpRock(r.http, r.cb, r.site))      throw new Error(        `Crawlbase ${r.http}, cb_status ${r.cb}: ${r.body.slice(0, 200)}`,      );    console.warn(`Crawlbase ${r.http}, cb_status ${r.cb}; trying UpRock`);  }  return { via: "uprock", markdown: await viaUpRock(url, country) };}​// node read.ts https://developer.mozilla.org/en-US/docs/Web/HTTP DEconst [url = "https://developer.mozilla.org/en-US/docs/Web/HTTP", country = "US"] =  process.argv.slice(2);const page = await read(url, country);console.log(page.via, page.markdown.slice(0, 200));

# pip install requests   env: CRAWLBASE_TOKEN, UPROCK_API_KEY# Calls Crawlbase's HTTP API directly: the Python SDK drops the headers, and with them# cb_status, on any non-2xx response, which is how a failed crawl arrives.import osimport sysfrom typing import Optionalfrom urllib.parse import urlparse​import requests​# Crawlbase reports each domain's price tier on billed responses (X-Domain-Complexity).# A domain billed at any other tier goes straight to UpRock from then on.ACCEPTED_TIERS = {"standard"}uprock_domains: set = set()  # save it somewhere if you want it to persist​# HTTP statuses worth a second network: a bad token (401), your spend cap or account# limit (402, 403), the concurrency limit (429), errors and timeouts (500, 503, 504).TRY_UPROCK_HTTP = {401, 402, 403, 429, 500, 503, 504}​​def worth_uprock(http: int, cb: int, site: int) -> bool:    # cb_status (crawlbase.com/docs/status-codes, checked 25 Oct 2026): 704 is the    # site's own error, worth a second try only if it's a 5xx; 595 is a click selector    # to fix; 503 (a block page), 705 (the site said 403), 706 (429) and other 5xx to    # 7xx codes are worth it.    if cb == 704:        return site >= 500    if cb == 595:        return False    return cb >= 500 or http in TRY_UPROCK_HTTP​​def via_uprock(url: str, country: str) -> str:    # One call: UpRock loads the page on a real device in that country, tries a fresh device    # every 7 s (up to 3 tries) and returns markdown. Failed calls aren't billed.    res = requests.post(        "https://edge.uprock.com/v1/crawl",        headers={"Authorization": f"Bearer {os.environ['UPROCK_API_KEY']}"},        json={"url": url, "country": country},        timeout=90,    )    page = res.json()    if not res.ok:        raise RuntimeError(f"UpRock {res.status_code} {page.get('code')}: {page.get('error')}")    return page.get("markdown", "")​​def via_crawlbase(url: str, country: str) -> Optional[dict]:    params = {        "token": os.environ.get("CRAWLBASE_TOKEN", ""),        "url": url,        "format": "md",        "country": country,    }    try:  # Crawlbase asks for a client timeout of at least 90 s        res = requests.get("https://api.crawlbase.com/", params=params, timeout=90)    except requests.RequestException as err:        print(f"Crawlbase unreachable ({err}); trying UpRock", file=sys.stderr)        return None    h = res.headers  # case-insensitive    return {        "http": res.status_code,        "cb": int(h.get("cb_status") or h.get("pc_status") or 0),  # pc_status: old name        "site": int(h.get("original_status") or 0),        "tier": h.get("x-domain-complexity") or h.get("domain_complexity") or "",        "body": res.text,    }​​def read(url: str, country: str = "US") -> dict:    domain = urlparse(url).hostname or ""    r = None if domain in uprock_domains else via_crawlbase(url, country)    if r:        if r["tier"] and r["tier"] not in ACCEPTED_TIERS:            # This page is billed already; the next ones from this domain aren't.            uprock_domains.add(domain)            print(f"Crawlbase bills {domain} as {r['tier']!r}", file=sys.stderr)        if r["http"] == 200 and r["cb"] == 200:            if r["site"] in (404, 410):                # The site's answer (and billed): a second crawler will see the same.                raise LookupError(f"The page is gone ({r['site']})")            return {"via": "crawlbase", "markdown": r["body"]}        if r["cb"] != 207 and not worth_uprock(r["http"], r["cb"], r["site"]):            raise RuntimeError(f"Crawlbase {r['http']}, cb_status {r['cb']}")        status = f"{r['http']}, cb_status {r['cb']}"        print(f"Crawlbase {status}; trying UpRock", file=sys.stderr)    return {"via": "uprock", "markdown": via_uprock(url, country)}​​if __name__ == "__main__":    # python read.py https://developer.mozilla.org/en-US/docs/Web/HTTP DE    url = sys.argv[1] if len(sys.argv) > 1 else "https://developer.mozilla.org/en-US/docs/Web/HTTP"    country = sys.argv[2] if len(sys.argv) > 2 else "US"    page = read(url, country)    print(page["via"], page["markdown"][:200])

// npm i crawlbase @modelcontextprotocol/sdk   (Node 18+)   env: CRAWLBASE_TOKEN, UPROCK_API_KEYimport { CrawlingAPI } from "crawlbase";import { Client } from "@modelcontextprotocol/sdk/client/index.js";import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";​// Crawlbase asks for a client timeout of at least 90 s.const crawlbase = new CrawlingAPI({  token: process.env.CRAWLBASE_TOKEN ?? "",  timeout: 90_000,});const uprock = new Client({ name: "crawlbase-fallback", version: "1.0.0" });await uprock.connect(  new StreamableHTTPClientTransport(    new URL("https://mcp.uprock.ai/mcp"),    { requestInit: { headers: { Authorization: `Bearer ${process.env.UPROCK_API_KEY}` } } },  ),);​// Crawlbase reports each domain's price tier on billed responses (X-Domain-Complexity).// A domain billed at any other tier goes straight to UpRock from then on.const ACCEPTED_TIERS = new Set(["standard"]);const uprockDomains = new Set<string>(); // save it somewhere if you want it to persist​// HTTP statuses worth a second network: a bad token (401), your spend cap or account// limit (402, 403), the concurrency limit (429), errors and timeouts (500, 503, 504).const TRY_UPROCK_HTTP = new Set([401, 402, 403, 429, 500, 503, 504]);​// cb_status (crawlbase.com/docs/status-codes, checked 25 Oct 2026): 704 is the site's own// error, worth a second try only if it's a 5xx; 595 is a click selector to fix; 503 (a// block page), 705 (the site said 403), 706 (429) and other 5xx to 7xx codes are worth it.function worthUpRock(http: number, cb: number, site: number) {  if (cb === 704) return site >= 500;  if (cb === 595) return false;  return cb >= 500 || TRY_UPROCK_HTTP.has(http);}​const textOf = (content: unknown) =>  (content as { type: string; text?: string }[]).find((c) => c.type === "text")?.text ??  "";​async function viaUpRock(url: string, country: string) {  const res = await uprock.callTool(    { name: "crawl_fetch", arguments: { url, country } },    undefined,    { timeout: 300_000 },  );  if (res.isError) throw new Error(`UpRock: ${textOf(res.content)}`);  const uri = JSON.parse(textOf(res.content)).content?.markdown?.resource;  if (!uri) throw new Error(`UpRock: no markdown for ${url}`);  return textOf(    (await uprock.callTool({ name: "resource_fetch", arguments: { uri } })).content,  );}​async function viaCrawlbase(url: string, country: string) {  try {    const res = await crawlbase.get(url, { format: "md", country });    const header = (name: string) => String(res.headers[name] ?? "");    return {      http: Number(res.statusCode),      cb: Number(header("cb_status") || header("pc_status")), // pc_status: the old name      site: Number(header("original_status")),      tier: header("x-domain-complexity") || header("domain_complexity"),      body: String(res.body),    };  } catch (err) {    console.warn(`Crawlbase unreachable or timed out (${err}); trying UpRock`);    return null;  }}​export async function read(url: string, country = "US") {  const domain = new URL(url).hostname;  const r = uprockDomains.has(domain) ? null : await viaCrawlbase(url, country);  if (r) {    if (r.tier && !ACCEPTED_TIERS.has(r.tier)) {      uprockDomains.add(domain); // this page is billed already; the next ones aren't      console.warn(        `Crawlbase bills ${domain} as "${r.tier}"; using UpRock from now on`,      );    }    if (r.http === 200 && r.cb === 200) {      // 404 and 410 are the site's answer (and billed): a second crawler will see the same.      if (r.site === 404 || r.site === 410)        throw new Error(`The page is gone (${r.site})`);      return { via: "crawlbase", markdown: r.body };    }    if (r.cb !== 207 && !worthUpRock(r.http, r.cb, r.site))      throw new Error(        `Crawlbase ${r.http}, cb_status ${r.cb}: ${r.body.slice(0, 200)}`,      );    console.warn(`Crawlbase ${r.http}, cb_status ${r.cb}; trying UpRock`);  }  return { via: "uprock", markdown: await viaUpRock(url, country) };}​// node read.ts https://developer.mozilla.org/en-US/docs/Web/HTTP DEconst [url = "https://developer.mozilla.org/en-US/docs/Web/HTTP", country = "US"] =  process.argv.slice(2);try {  const page = await read(url, country);  console.log(page.via, page.markdown.slice(0, 200));} finally {  await uprock.close();}

# pip install requests; npm i -g uprock   env: CRAWLBASE_TOKEN, UPROCK_API_KEY# Calls Crawlbase's HTTP API directly: the Python SDK drops the headers, and with them# cb_status, on any non-2xx response, which is how a failed crawl arrives.import jsonimport osimport subprocessimport sysfrom typing import Optionalfrom urllib.parse import urlparse​import requests​# Crawlbase reports each domain's price tier on billed responses (X-Domain-Complexity).# A domain billed at any other tier goes straight to UpRock from then on.ACCEPTED_TIERS = {"standard"}uprock_domains: set = set()  # save it somewhere if you want it to persist​# HTTP statuses worth a second network: a bad token (401), your spend cap or account# limit (402, 403), the concurrency limit (429), errors and timeouts (500, 503, 504).TRY_UPROCK_HTTP = {401, 402, 403, 429, 500, 503, 504}​​def worth_uprock(http: int, cb: int, site: int) -> bool:    # cb_status (crawlbase.com/docs/status-codes, checked 25 Oct 2026): 704 is the    # site's own error, worth a second try only if it's a 5xx; 595 is a click selector    # to fix; 503 (a block page), 705 (the site said 403), 706 (429) and other 5xx to    # 7xx codes are worth it.    if cb == 704:        return site >= 500    if cb == 595:        return False    return cb >= 500 or http in TRY_UPROCK_HTTP​​def via_uprock(url: str, country: str) -> str:    run = subprocess.run(        ["uprock", "ai", "crawl", url, "--country", country, "--content"],        capture_output=True,        text=True,        timeout=330,    )    if run.returncode != 0:        raise RuntimeError(f"UpRock: {run.stderr.strip()}")    return json.loads(run.stdout).get("inlined_markdown", "")​​def via_crawlbase(url: str, country: str) -> Optional[dict]:    params = {        "token": os.environ.get("CRAWLBASE_TOKEN", ""),        "url": url,        "format": "md",        "country": country,    }    try:  # Crawlbase asks for a client timeout of at least 90 s        res = requests.get("https://api.crawlbase.com/", params=params, timeout=90)    except requests.RequestException as err:        print(f"Crawlbase unreachable ({err}); trying UpRock", file=sys.stderr)        return None    h = res.headers  # case-insensitive    return {        "http": res.status_code,        "cb": int(h.get("cb_status") or h.get("pc_status") or 0),  # pc_status: old name        "site": int(h.get("original_status") or 0),        "tier": h.get("x-domain-complexity") or h.get("domain_complexity") or "",        "body": res.text,    }​​def read(url: str, country: str = "US") -> dict:    domain = urlparse(url).hostname or ""    r = None if domain in uprock_domains else via_crawlbase(url, country)    if r:        if r["tier"] and r["tier"] not in ACCEPTED_TIERS:            # This page is billed already; the next ones from this domain aren't.            uprock_domains.add(domain)            print(f"Crawlbase bills {domain} as {r['tier']!r}", file=sys.stderr)        if r["http"] == 200 and r["cb"] == 200:            if r["site"] in (404, 410):                # The site's answer (and billed): a second crawler will see the same.                raise LookupError(f"The page is gone ({r['site']})")            return {"via": "crawlbase", "markdown": r["body"]}        if r["cb"] != 207 and not worth_uprock(r["http"], r["cb"], r["site"]):            raise RuntimeError(f"Crawlbase {r['http']}, cb_status {r['cb']}")        status = f"{r['http']}, cb_status {r['cb']}"        print(f"Crawlbase {status}; trying UpRock", file=sys.stderr)    return {"via": "uprock", "markdown": via_uprock(url, country)}​​if __name__ == "__main__":    # python read.py https://developer.mozilla.org/en-US/docs/Web/HTTP DE    url = sys.argv[1] if len(sys.argv) > 1 else "https://developer.mozilla.org/en-US/docs/Web/HTTP"    country = sys.argv[2] if len(sys.argv) > 2 else "US"    page = read(url, country)    print(page["via"], page["markdown"][:200])