{
 "generatedAt": "2026-11-05T11:04:24+00:00",
 "probedAt": "2026-11-05T11:00:58.823Z",
 "vantage": "one EU datacenter IP, no proxy; best of two attempts per check",
 "summary": {
  "sites": 100,
  "grades": {
   "A": 30,
   "B": 16,
   "C": 13,
   "D": 20,
   "F": 21
  },
  "browserBlocked": 42,
  "httpBlocked": 47,
  "captcha": 31,
  "cookieWall": 7,
  "cookieBanner": 13,
  "robotsBlocksUser": 26,
  "robotsBlocksTraining": 40,
  "llmsTxt": 21,
  "markdown": 6,
  "byTier": {
   "Giant": {
    "n": 25,
    "browserBlocked": 9,
    "robotsBlocksUser": 13,
    "avg": 53
   },
   "Major": {
    "n": 25,
    "browserBlocked": 11,
    "robotsBlocksUser": 4,
    "avg": 53
   },
   "Mid": {
    "n": 25,
    "browserBlocked": 12,
    "robotsBlocksUser": 6,
    "avg": 57
   },
   "Niche": {
    "n": 25,
    "browserBlocked": 10,
    "robotsBlocksUser": 3,
    "avg": 65
   }
  }
 },
 "sites": [
  {
   "rank_popularity": 1,
   "domain": "google.com",
   "url": "https://www.google.com/",
   "category": "Search",
   "tier": "Giant",
   "score": 60,
   "grade": "C",
   "points": {
    "robots": 30.0,
    "http": 0,
    "browser": 30,
    "extras": 0
   },
   "notes": [
    [
     "agent fetch: JavaScript-only shell",
     "warn"
    ],
    [
     "cookie wall blocks the page",
     "bad"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": false
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Google",
    "cookie": "wall"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 50
  },
  {
   "rank_popularity": 2,
   "domain": "youtube.com",
   "url": "https://www.youtube.com/",
   "category": "Video",
   "tier": "Giant",
   "score": 70,
   "grade": "B",
   "points": {
    "robots": 30.0,
    "http": 0,
    "browser": 40,
    "extras": 0
   },
   "notes": [
    [
     "agent fetch: JavaScript-only shell",
     "warn"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": false
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "YouTube",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 40
  },
  {
   "rank_popularity": 3,
   "domain": "facebook.com",
   "url": "https://www.facebook.com/",
   "category": "Social",
   "tier": "Giant",
   "score": 60,
   "grade": "C",
   "points": {
    "robots": 0,
    "http": 20,
    "browser": 40,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks user-driven agents",
     "bad"
    ],
    [
     "robots: blocks AI search",
     "bad"
    ],
    [
     "robots: blocks 6/6 training crawlers",
     "warn"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": false,
    "Claude-User": false,
    "Perplexity-User": false,
    "OAI-SearchBot": false,
    "Claude-SearchBot": false,
    "PerplexityBot": false,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": false,
    "CCBot": false,
    "Applebot-Extended": false,
    "meta-externalagent": false
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Facebook",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 51
  },
  {
   "rank_popularity": 4,
   "domain": "instagram.com",
   "url": "https://www.instagram.com/",
   "category": "Social",
   "tier": "Giant",
   "score": 60,
   "grade": "C",
   "points": {
    "robots": 0,
    "http": 20,
    "browser": 40,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks user-driven agents",
     "bad"
    ],
    [
     "robots: blocks AI search",
     "bad"
    ],
    [
     "robots: blocks 6/6 training crawlers",
     "warn"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": false,
    "Claude-User": false,
    "Perplexity-User": false,
    "OAI-SearchBot": false,
    "Claude-SearchBot": false,
    "PerplexityBot": false,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": false,
    "CCBot": false,
    "Applebot-Extended": false,
    "meta-externalagent": false
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Instagram",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 52
  },
  {
   "rank_popularity": 5,
   "domain": "x.com",
   "url": "https://x.com/",
   "category": "Social",
   "tier": "Giant",
   "score": 0,
   "grade": "F",
   "points": {
    "robots": 0,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks user-driven agents",
     "bad"
    ],
    [
     "robots: blocks AI search",
     "bad"
    ],
    [
     "robots: blocks 6/6 training crawlers",
     "warn"
    ],
    [
     "agent fetch: HTTP 402",
     "bad"
    ],
    [
     "browser: HTTP 403",
     "bad"
    ]
   ],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": false,
    "Claude-User": false,
    "Perplexity-User": false,
    "OAI-SearchBot": false,
    "Claude-SearchBot": false,
    "PerplexityBot": false,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": false,
    "CCBot": false,
    "Applebot-Extended": false,
    "meta-externalagent": false
   },
   "robotsFound": true,
   "http": {
    "status": 402,
    "ok": false
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 97
  },
  {
   "rank_popularity": 6,
   "domain": "wikipedia.org",
   "url": "https://www.wikipedia.org/",
   "category": "Reference",
   "tier": "Giant",
   "score": 90,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 40,
    "extras": 0
   },
   "notes": [],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Wikipedia",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 11
  },
  {
   "rank_popularity": 7,
   "domain": "reddit.com",
   "url": "https://www.reddit.com/",
   "category": "Community",
   "tier": "Giant",
   "score": 0,
   "grade": "F",
   "points": {
    "robots": 0,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks user-driven agents",
     "bad"
    ],
    [
     "robots: blocks AI search",
     "bad"
    ],
    [
     "robots: blocks 6/6 training crawlers",
     "warn"
    ],
    [
     "agent fetch: HTTP 403",
     "bad"
    ],
    [
     "browser: HTTP 403",
     "bad"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": false,
    "Claude-User": false,
    "Perplexity-User": false,
    "OAI-SearchBot": false,
    "Claude-SearchBot": false,
    "PerplexityBot": false,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": false,
    "CCBot": false,
    "Applebot-Extended": false,
    "meta-externalagent": false
   },
   "robotsFound": true,
   "http": {
    "status": 403,
    "ok": false
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 98
  },
  {
   "rank_popularity": 8,
   "domain": "amazon.com",
   "url": "https://www.amazon.com/",
   "category": "Shopping",
   "tier": "Giant",
   "score": 42,
   "grade": "D",
   "points": {
    "robots": 1.5,
    "http": 0,
    "browser": 40,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks user-driven agents",
     "bad"
    ],
    [
     "robots: blocks AI search",
     "bad"
    ],
    [
     "robots: blocks 5/6 training crawlers",
     "warn"
    ],
    [
     "agent fetch: JavaScript-only shell",
     "warn"
    ]
   ],
   "vendors": [
    "Akamai"
   ],
   "robots": {
    "ChatGPT-User": false,
    "Claude-User": false,
    "Perplexity-User": false,
    "OAI-SearchBot": false,
    "Claude-SearchBot": false,
    "PerplexityBot": false,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": false,
    "CCBot": false,
    "Applebot-Extended": true,
    "meta-externalagent": false
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": false
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Amazon.com. Spend less. Smile more.",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 63
  },
  {
   "rank_popularity": 9,
   "domain": "linkedin.com",
   "url": "https://www.linkedin.com/",
   "category": "Social",
   "tier": "Giant",
   "score": 28,
   "grade": "F",
   "points": {
    "robots": 7.5,
    "http": 20,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks user-driven agents",
     "bad"
    ],
    [
     "robots: blocks AI search",
     "bad"
    ],
    [
     "robots: blocks 5/6 training crawlers",
     "warn"
    ],
    [
     "browser: Cloudflare challenge",
     "bad"
    ]
   ],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": false,
    "Claude-User": false,
    "Perplexity-User": false,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": false,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": false,
    "CCBot": false,
    "Applebot-Extended": true,
    "meta-externalagent": false
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "Attention Required! | Cloudflare",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 80
  },
  {
   "rank_popularity": 10,
   "domain": "tiktok.com",
   "url": "https://www.tiktok.com/",
   "category": "Video",
   "tier": "Giant",
   "score": 0,
   "grade": "F",
   "points": {
    "robots": 0,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks user-driven agents",
     "bad"
    ],
    [
     "robots: blocks AI search",
     "bad"
    ],
    [
     "robots: blocks 6/6 training crawlers",
     "warn"
    ],
    [
     "agent fetch: JavaScript-only shell",
     "warn"
    ],
    [
     "browser: page came back empty",
     "bad"
    ]
   ],
   "vendors": [
    "Akamai"
   ],
   "robots": {
    "ChatGPT-User": false,
    "Claude-User": false,
    "Perplexity-User": false,
    "OAI-SearchBot": false,
    "Claude-SearchBot": false,
    "PerplexityBot": false,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": false,
    "CCBot": false,
    "Applebot-Extended": false,
    "meta-externalagent": false
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": false
   },
   "browser": {
    "status": 200,
    "ok": false,
    "title": "TikTok - Make Your Day",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 99
  },
  {
   "rank_popularity": 11,
   "domain": "netflix.com",
   "url": "https://www.netflix.com/",
   "category": "Video",
   "tier": "Giant",
   "score": 78,
   "grade": "B",
   "points": {
    "robots": 23.0,
    "http": 20,
    "browser": 35,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks user-driven agents",
     "bad"
    ],
    [
     "robots: blocks 2/6 training crawlers",
     "warn"
    ],
    [
     "cookie banner",
     "warn"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": false,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": false,
    "Applebot-Extended": true,
    "meta-externalagent": false
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Netflix - Watch TV Shows Online, Watch Movies Online",
    "cookie": "banner"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 36
  },
  {
   "rank_popularity": 12,
   "domain": "bing.com",
   "url": "https://www.bing.com/",
   "category": "Search",
   "tier": "Giant",
   "score": 90,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 40,
    "extras": 0
   },
   "notes": [],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Search - Microsoft Bing",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 12
  },
  {
   "rank_popularity": 13,
   "domain": "yahoo.com",
   "url": "https://www.yahoo.com/",
   "category": "Portal",
   "tier": "Giant",
   "score": 72,
   "grade": "B",
   "points": {
    "robots": 17.0,
    "http": 20,
    "browser": 35,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks user-driven agents",
     "bad"
    ],
    [
     "robots: blocks AI search",
     "bad"
    ],
    [
     "robots: blocks 4/6 training crawlers",
     "warn"
    ],
    [
     "cookie banner",
     "warn"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": false,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": false,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": false,
    "CCBot": false,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Yahoo | Mail, Weather, Search, Politics, News, Finance, Sports & Videos",
    "cookie": "banner"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 39
  },
  {
   "rank_popularity": 14,
   "domain": "chatgpt.com",
   "url": "https://chatgpt.com/",
   "category": "AI",
   "tier": "Giant",
   "score": 44,
   "grade": "D",
   "points": {
    "robots": 24.0,
    "http": 20,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks AI search",
     "bad"
    ],
    [
     "robots: blocks 2/6 training crawlers",
     "warn"
    ],
    [
     "browser: Cloudflare challenge",
     "bad"
    ]
   ],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": false,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": false,
    "CCBot": false,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "Just a moment...",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 61
  },
  {
   "rank_popularity": 15,
   "domain": "microsoft.com",
   "url": "https://www.microsoft.com/",
   "category": "Tech",
   "tier": "Giant",
   "score": 50,
   "grade": "C",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "browser: timed out",
     "bad"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": null,
    "ok": false,
    "title": "",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 59
  },
  {
   "rank_popularity": 16,
   "domain": "apple.com",
   "url": "https://www.apple.com/",
   "category": "Tech",
   "tier": "Giant",
   "score": 90,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 40,
    "extras": 0
   },
   "notes": [],
   "vendors": [
    "Akamai"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Apple",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 13
  },
  {
   "rank_popularity": 17,
   "domain": "ebay.com",
   "url": "https://www.ebay.com/",
   "category": "Shopping",
   "tier": "Giant",
   "score": 30,
   "grade": "D",
   "points": {
    "robots": 30.0,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "agent fetch: connection refused/timeout",
     "bad"
    ],
    [
     "browser: Block page: Error Page | eBay",
     "bad"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": false,
   "http": {
    "status": 0,
    "ok": false
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "Error Page | eBay",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 69
  },
  {
   "rank_popularity": 18,
   "domain": "twitch.tv",
   "url": "https://twitch.tv/",
   "category": "Video",
   "tier": "Giant",
   "score": 70,
   "grade": "B",
   "points": {
    "robots": 30.0,
    "http": 0,
    "browser": 40,
    "extras": 0
   },
   "notes": [
    [
     "agent fetch: JavaScript-only shell",
     "warn"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": false
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Twitch",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 41
  },
  {
   "rank_popularity": 19,
   "domain": "pinterest.com",
   "url": "https://www.pinterest.com/",
   "category": "Social",
   "tier": "Giant",
   "score": 52,
   "grade": "C",
   "points": {
    "robots": 1.5,
    "http": 20,
    "browser": 30,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks user-driven agents",
     "bad"
    ],
    [
     "robots: blocks AI search",
     "bad"
    ],
    [
     "robots: blocks 5/6 training crawlers",
     "warn"
    ],
    [
     "cookie wall blocks the page",
     "bad"
    ]
   ],
   "vendors": [
    "Akamai"
   ],
   "robots": {
    "ChatGPT-User": false,
    "Claude-User": false,
    "Perplexity-User": false,
    "OAI-SearchBot": false,
    "Claude-SearchBot": false,
    "PerplexityBot": false,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": false,
    "CCBot": false,
    "Applebot-Extended": true,
    "meta-externalagent": false
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Pinterest",
    "cookie": "wall"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 58
  },
  {
   "rank_popularity": 20,
   "domain": "whatsapp.com",
   "url": "https://www.whatsapp.com/",
   "category": "Messaging",
   "tier": "Giant",
   "score": 81,
   "grade": "B",
   "points": {
    "robots": 21.0,
    "http": 20,
    "browser": 40,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks AI search",
     "bad"
    ],
    [
     "robots: blocks 4/6 training crawlers",
     "warn"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": false,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": false,
    "CCBot": true,
    "Applebot-Extended": false,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "WhatsApp | Secure and Reliable Free Private Messaging and Calling",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 33
  },
  {
   "rank_popularity": 21,
   "domain": "walmart.com",
   "url": "https://www.walmart.com/",
   "category": "Shopping",
   "tier": "Giant",
   "score": 90,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 40,
    "extras": 0
   },
   "notes": [],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Walmart | Save Money. Live better.",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 14
  },
  {
   "rank_popularity": 22,
   "domain": "nytimes.com",
   "url": "https://www.nytimes.com/",
   "category": "News",
   "tier": "Giant",
   "score": 0,
   "grade": "F",
   "points": {
    "robots": 0,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks user-driven agents",
     "bad"
    ],
    [
     "robots: blocks AI search",
     "bad"
    ],
    [
     "robots: blocks 6/6 training crawlers",
     "warn"
    ],
    [
     "agent fetch: HTTP 403",
     "bad"
    ],
    [
     "browser: DataDome CAPTCHA, CAPTCHA iframe",
     "bad"
    ]
   ],
   "vendors": [
    "DataDome"
   ],
   "robots": {
    "ChatGPT-User": false,
    "Claude-User": false,
    "Perplexity-User": false,
    "OAI-SearchBot": false,
    "Claude-SearchBot": false,
    "PerplexityBot": false,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": false,
    "CCBot": false,
    "Applebot-Extended": false,
    "meta-externalagent": false
   },
   "robotsFound": true,
   "http": {
    "status": 403,
    "ok": false
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "nytimes.com",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 100
  },
  {
   "rank_popularity": 23,
   "domain": "cnn.com",
   "url": "https://www.cnn.com/",
   "category": "News",
   "tier": "Giant",
   "score": 3,
   "grade": "F",
   "points": {
    "robots": 3.0,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks user-driven agents",
     "bad"
    ],
    [
     "robots: blocks AI search",
     "bad"
    ],
    [
     "robots: blocks 4/6 training crawlers",
     "warn"
    ],
    [
     "agent fetch: HTTP 451",
     "bad"
    ],
    [
     "browser: page came back empty",
     "bad"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": false,
    "Claude-User": false,
    "Perplexity-User": false,
    "OAI-SearchBot": false,
    "Claude-SearchBot": false,
    "PerplexityBot": false,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": true,
    "CCBot": false,
    "Applebot-Extended": false,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 451,
    "ok": false
   },
   "browser": {
    "status": 200,
    "ok": false,
    "title": "",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 95
  },
  {
   "rank_popularity": 24,
   "domain": "bbc.com",
   "url": "https://www.bbc.com/",
   "category": "News",
   "tier": "Giant",
   "score": 59,
   "grade": "C",
   "points": {
    "robots": 4,
    "http": 20,
    "browser": 35,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks user-driven agents",
     "bad"
    ],
    [
     "robots: blocks AI search",
     "bad"
    ],
    [
     "robots: blocks 6/6 training crawlers",
     "warn"
    ],
    [
     "cookie banner",
     "warn"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": false,
    "Claude-User": true,
    "Perplexity-User": false,
    "OAI-SearchBot": false,
    "Claude-SearchBot": false,
    "PerplexityBot": false,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": false,
    "CCBot": false,
    "Applebot-Extended": false,
    "meta-externalagent": false
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "BBC Home | Breaking News, World News, US News, Sports, Business, Innovation, Climate, Culture, Travel, Video & Audio",
    "cookie": "banner"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 55
  },
  {
   "rank_popularity": 25,
   "domain": "github.com",
   "url": "https://github.com/",
   "category": "Developer",
   "tier": "Giant",
   "score": 95,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 40,
    "extras": 5
   },
   "notes": [
    [
     "llms.txt",
     "good"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "GitHub \u00b7 Change is constant. GitHub keeps you ahead. \u00b7 GitHub \u00b7 GitHub",
    "cookie": "none"
   },
   "llmsTxt": true,
   "markdown": false,
   "rank_agent": 3
  },
  {
   "rank_popularity": 26,
   "domain": "stackoverflow.com",
   "url": "https://www.stackoverflow.com/",
   "category": "Developer",
   "tier": "Major",
   "score": 30,
   "grade": "D",
   "points": {
    "robots": 30.0,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "agent fetch: HTTP 402",
     "bad"
    ],
    [
     "browser: Cloudflare challenge",
     "bad"
    ]
   ],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": false,
   "http": {
    "status": 402,
    "ok": false
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "Just a moment...",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 70
  },
  {
   "rank_popularity": 27,
   "domain": "quora.com",
   "url": "https://www.quora.com/",
   "category": "Community",
   "tier": "Major",
   "score": 8,
   "grade": "F",
   "points": {
    "robots": 8.5,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks user-driven agents",
     "bad"
    ],
    [
     "robots: blocks AI search",
     "bad"
    ],
    [
     "robots: blocks 3/6 training crawlers",
     "warn"
    ],
    [
     "agent fetch: Cloudflare challenge",
     "bad"
    ],
    [
     "browser: Cloudflare challenge",
     "bad"
    ]
   ],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": false,
    "Claude-User": false,
    "Perplexity-User": true,
    "OAI-SearchBot": false,
    "Claude-SearchBot": false,
    "PerplexityBot": false,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": false,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 403,
    "ok": false
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "Just a moment...",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 92
  },
  {
   "rank_popularity": 28,
   "domain": "imdb.com",
   "url": "https://www.imdb.com/",
   "category": "Entertainment",
   "tier": "Major",
   "score": 2,
   "grade": "F",
   "points": {
    "robots": 1.5,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks user-driven agents",
     "bad"
    ],
    [
     "robots: blocks AI search",
     "bad"
    ],
    [
     "robots: blocks 5/6 training crawlers",
     "warn"
    ],
    [
     "agent fetch: Block page: 403 Forbidden",
     "bad"
    ],
    [
     "browser: Block page: 403 Forbidden",
     "bad"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": false,
    "Claude-User": false,
    "Perplexity-User": false,
    "OAI-SearchBot": false,
    "Claude-SearchBot": false,
    "PerplexityBot": false,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": false,
    "CCBot": false,
    "Applebot-Extended": true,
    "meta-externalagent": false
   },
   "robotsFound": true,
   "http": {
    "status": 403,
    "ok": false
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "403 Forbidden",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 96
  },
  {
   "rank_popularity": 29,
   "domain": "espn.com",
   "url": "https://www.espn.com/",
   "category": "Sports",
   "tier": "Major",
   "score": 22,
   "grade": "F",
   "points": {
    "robots": 21.5,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks user-driven agents",
     "bad"
    ],
    [
     "robots: blocks 3/6 training crawlers",
     "warn"
    ],
    [
     "agent fetch: Block page: ERROR: The request could not be satisfied",
     "bad"
    ],
    [
     "browser: Block page: ERROR: The request could not be satisfied",
     "bad"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": false,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": false,
    "ClaudeBot": true,
    "Google-Extended": false,
    "CCBot": false,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 403,
    "ok": false
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "ERROR: The request could not be satisfied",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 85
  },
  {
   "rank_popularity": 30,
   "domain": "weather.com",
   "url": "https://www.weather.com/",
   "category": "Weather",
   "tier": "Major",
   "score": 84,
   "grade": "B",
   "points": {
    "robots": 28.5,
    "http": 20,
    "browser": 30,
    "extras": 5
   },
   "notes": [
    [
     "robots: blocks 1/6 training crawlers",
     "warn"
    ],
    [
     "cookie wall blocks the page",
     "bad"
    ],
    [
     "llms.txt",
     "good"
    ]
   ],
   "vendors": [
    "Akamai"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": false,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "National and Local Weather Radar, Daily Forecast, Hurricane and information from The Weather Channel and weather.com",
    "cookie": "wall"
   },
   "llmsTxt": true,
   "markdown": false,
   "rank_agent": 31
  },
  {
   "rank_popularity": 31,
   "domain": "zillow.com",
   "url": "https://www.zillow.com/",
   "category": "Real estate",
   "tier": "Major",
   "score": 27,
   "grade": "F",
   "points": {
    "robots": 27.0,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks 2/6 training crawlers",
     "warn"
    ],
    [
     "agent fetch: HUMAN/PerimeterX CAPTCHA",
     "bad"
    ],
    [
     "browser: HUMAN/PerimeterX CAPTCHA",
     "bad"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": false,
    "meta-externalagent": false
   },
   "robotsFound": true,
   "http": {
    "status": 403,
    "ok": false
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "Access to this page has been denied",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 81
  },
  {
   "rank_popularity": 32,
   "domain": "etsy.com",
   "url": "https://www.etsy.com/",
   "category": "Shopping",
   "tier": "Major",
   "score": 35,
   "grade": "D",
   "points": {
    "robots": 30.0,
    "http": 0,
    "browser": 0,
    "extras": 5
   },
   "notes": [
    [
     "agent fetch: DataDome CAPTCHA",
     "bad"
    ],
    [
     "browser: DataDome CAPTCHA, CAPTCHA iframe",
     "bad"
    ],
    [
     "llms.txt",
     "good"
    ]
   ],
   "vendors": [
    "DataDome"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 403,
    "ok": false
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "etsy.com",
    "cookie": "none"
   },
   "llmsTxt": true,
   "markdown": false,
   "rank_agent": 65
  },
  {
   "rank_popularity": 33,
   "domain": "booking.com",
   "url": "https://www.booking.com/",
   "category": "Travel",
   "tier": "Major",
   "score": 70,
   "grade": "B",
   "points": {
    "robots": 30.0,
    "http": 0,
    "browser": 40,
    "extras": 0
   },
   "notes": [
    [
     "agent fetch: AWS WAF challenge, CAPTCHA wall",
     "bad"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 202,
    "ok": false
   },
   "browser": {
    "status": 202,
    "ok": true,
    "title": "Booking.com | Official site | The best hotels, flights, car rentals & accommodations",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 42
  },
  {
   "rank_popularity": 34,
   "domain": "tripadvisor.com",
   "url": "https://www.tripadvisor.com/",
   "category": "Travel",
   "tier": "Major",
   "score": 21,
   "grade": "F",
   "points": {
    "robots": 21,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks 6/6 training crawlers",
     "warn"
    ],
    [
     "agent fetch: DataDome CAPTCHA",
     "bad"
    ],
    [
     "browser: DataDome CAPTCHA, CAPTCHA iframe",
     "bad"
    ]
   ],
   "vendors": [
    "DataDome"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": false,
    "CCBot": false,
    "Applebot-Extended": false,
    "meta-externalagent": false
   },
   "robotsFound": true,
   "http": {
    "status": 403,
    "ok": false
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "tripadvisor.com",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 87
  },
  {
   "rank_popularity": 35,
   "domain": "airbnb.com",
   "url": "https://www.airbnb.com/",
   "category": "Travel",
   "tier": "Major",
   "score": 60,
   "grade": "C",
   "points": {
    "robots": 25.5,
    "http": 0,
    "browser": 35,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks 3/6 training crawlers",
     "warn"
    ],
    [
     "agent fetch: Akamai block",
     "bad"
    ],
    [
     "cookie banner",
     "warn"
    ]
   ],
   "vendors": [
    "Akamai"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": false,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 403,
    "ok": false
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Airbnb | Vacation rentals, cabins, beach houses, & more",
    "cookie": "banner"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 53
  },
  {
   "rank_popularity": 36,
   "domain": "indeed.com",
   "url": "https://www.indeed.com/",
   "category": "Jobs",
   "tier": "Major",
   "score": 30,
   "grade": "D",
   "points": {
    "robots": 30.0,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "agent fetch: Cloudflare challenge",
     "bad"
    ],
    [
     "browser: Block page: Blocked - Indeed.com",
     "bad"
    ]
   ],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 403,
    "ok": false
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "Blocked - Indeed.com",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 71
  },
  {
   "rank_popularity": 37,
   "domain": "target.com",
   "url": "https://www.target.com/",
   "category": "Shopping",
   "tier": "Major",
   "score": 95,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 40,
    "extras": 5
   },
   "notes": [
    [
     "llms.txt",
     "good"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Target : Expect More. Pay Less.",
    "cookie": "none"
   },
   "llmsTxt": true,
   "markdown": false,
   "rank_agent": 4
  },
  {
   "rank_popularity": 38,
   "domain": "bestbuy.com",
   "url": "https://www.bestbuy.com/?intl=nosplash",
   "category": "Shopping",
   "tier": "Major",
   "score": 70,
   "grade": "B",
   "points": {
    "robots": 30.0,
    "http": 0,
    "browser": 40,
    "extras": 0
   },
   "notes": [
    [
     "agent fetch: connection refused/timeout",
     "bad"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": false,
   "http": {
    "status": 0,
    "ok": false
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Best Buy | Official Online Store | Shop Now & Save",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 43
  },
  {
   "rank_popularity": 39,
   "domain": "homedepot.com",
   "url": "https://www.homedepot.com/",
   "category": "Shopping",
   "tier": "Major",
   "score": 30,
   "grade": "D",
   "points": {
    "robots": 30.0,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "agent fetch: Akamai block",
     "bad"
    ],
    [
     "browser: Akamai block",
     "bad"
    ]
   ],
   "vendors": [
    "Akamai"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": false,
   "http": {
    "status": 403,
    "ok": false
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "Access Denied",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 72
  },
  {
   "rank_popularity": 40,
   "domain": "spotify.com",
   "url": "https://www.spotify.com/",
   "category": "Music",
   "tier": "Major",
   "score": 70,
   "grade": "B",
   "points": {
    "robots": 30.0,
    "http": 0,
    "browser": 40,
    "extras": 0
   },
   "notes": [
    [
     "agent fetch: JavaScript-only shell",
     "warn"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": false
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Spotify - Web Player: Listen to Music, Podcasts, and Audiobooks Online",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 44
  },
  {
   "rank_popularity": 41,
   "domain": "discord.com",
   "url": "https://www.discord.com/",
   "category": "Messaging",
   "tier": "Major",
   "score": 90,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 40,
    "extras": 0
   },
   "notes": [],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Discord - Group Chat That\u2019s All Fun & Games",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 15
  },
  {
   "rank_popularity": 42,
   "domain": "paypal.com",
   "url": "https://www.paypal.com/",
   "category": "Finance",
   "tier": "Major",
   "score": 85,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 30,
    "extras": 5
   },
   "notes": [
    [
     "cookie wall blocks the page",
     "bad"
    ],
    [
     "llms.txt",
     "good"
    ]
   ],
   "vendors": [
    "DataDome"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Compte PayPal - Paiements et Transferts d'argent | PayPal FR",
    "cookie": "wall"
   },
   "llmsTxt": true,
   "markdown": false,
   "rank_agent": 29
  },
  {
   "rank_popularity": 43,
   "domain": "craigslist.org",
   "url": "https://www.craigslist.org/",
   "category": "Classifieds",
   "tier": "Major",
   "score": 90,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 40,
    "extras": 0
   },
   "notes": [],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "craigslist: strasbourg jobs, apartments, for sale, services, community, and events",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 16
  },
  {
   "rank_popularity": 44,
   "domain": "theguardian.com",
   "url": "https://www.theguardian.com/",
   "category": "News",
   "tier": "Major",
   "score": 44,
   "grade": "D",
   "points": {
    "robots": 14.0,
    "http": 0,
    "browser": 30,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks user-driven agents",
     "bad"
    ],
    [
     "robots: blocks AI search",
     "bad"
    ],
    [
     "robots: blocks 4/6 training crawlers",
     "warn"
    ],
    [
     "agent fetch: HTTP 403",
     "bad"
    ],
    [
     "cookie wall blocks the page",
     "bad"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": false,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": false,
    "PerplexityBot": false,
    "GPTBot": true,
    "ClaudeBot": false,
    "Google-Extended": true,
    "CCBot": false,
    "Applebot-Extended": false,
    "meta-externalagent": false
   },
   "robotsFound": true,
   "http": {
    "status": 403,
    "ok": false
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Latest news, sport and opinion from the Guardian",
    "cookie": "wall"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 62
  },
  {
   "rank_popularity": 45,
   "domain": "medium.com",
   "url": "https://www.medium.com/",
   "category": "Publishing",
   "tier": "Major",
   "score": 24,
   "grade": "F",
   "points": {
    "robots": 24.0,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks 4/6 training crawlers",
     "warn"
    ],
    [
     "agent fetch: Cloudflare challenge",
     "bad"
    ],
    [
     "browser: Cloudflare challenge",
     "bad"
    ]
   ],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": false,
    "meta-externalagent": false
   },
   "robotsFound": true,
   "http": {
    "status": 403,
    "ok": false
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "Just a moment...",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 83
  },
  {
   "rank_popularity": 46,
   "domain": "wordpress.com",
   "url": "https://www.wordpress.com/",
   "category": "Publishing",
   "tier": "Major",
   "score": 65,
   "grade": "C",
   "points": {
    "robots": 30.0,
    "http": 0,
    "browser": 35,
    "extras": 0
   },
   "notes": [
    [
     "agent fetch: HTTP 403",
     "bad"
    ],
    [
     "cookie banner",
     "warn"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": false,
   "http": {
    "status": 403,
    "ok": false
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "WordPress.com: Everything You Need to Build Your Website",
    "cookie": "banner"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 47
  },
  {
   "rank_popularity": 47,
   "domain": "fandom.com",
   "url": "https://www.fandom.com/",
   "category": "Entertainment",
   "tier": "Major",
   "score": 30,
   "grade": "D",
   "points": {
    "robots": 30.0,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "agent fetch: HTTP 402",
     "bad"
    ],
    [
     "browser: Cloudflare challenge",
     "bad"
    ]
   ],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": false,
   "http": {
    "status": 402,
    "ok": false
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "Just a moment...",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 73
  },
  {
   "rank_popularity": 48,
   "domain": "nike.com",
   "url": "https://www.nike.com/",
   "category": "Shopping",
   "tier": "Major",
   "score": 70,
   "grade": "B",
   "points": {
    "robots": 30.0,
    "http": 0,
    "browser": 40,
    "extras": 0
   },
   "notes": [
    [
     "agent fetch: Akamai block",
     "bad"
    ]
   ],
   "vendors": [
    "Akamai"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": false,
   "http": {
    "status": 403,
    "ok": false
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Nike. Just Do It. Nike FR",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 45
  },
  {
   "rank_popularity": 49,
   "domain": "aliexpress.com",
   "url": "https://www.aliexpress.com/",
   "category": "Shopping",
   "tier": "Major",
   "score": 79,
   "grade": "B",
   "points": {
    "robots": 24.0,
    "http": 20,
    "browser": 35,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks 4/6 training crawlers",
     "warn"
    ],
    [
     "cookie banner",
     "warn"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": false,
    "CCBot": false,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "AliExpress France - Livraison gratuite et \u00e9conomisez plus - Le plus grand catalogue de choix d'achat en ligne pour l'\u00e9lectronique, mode, mai",
    "cookie": "banner"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 35
  },
  {
   "rank_popularity": 50,
   "domain": "shopify.com",
   "url": "https://www.shopify.com/",
   "category": "Commerce",
   "tier": "Major",
   "score": 95,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 40,
    "extras": 5
   },
   "notes": [
    [
     "llms.txt",
     "good"
    ]
   ],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Shopify: The All-in-One Commerce Platform for Businesses - Shopify",
    "cookie": "none"
   },
   "llmsTxt": true,
   "markdown": false,
   "rank_agent": 5
  },
  {
   "rank_popularity": 51,
   "domain": "cloudflare.com",
   "url": "https://www.cloudflare.com/",
   "category": "Tech",
   "tier": "Mid",
   "score": 95,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 35,
    "extras": 10
   },
   "notes": [
    [
     "cookie banner",
     "warn"
    ],
    [
     "llms.txt",
     "good"
    ],
    [
     "serves Markdown",
     "good"
    ]
   ],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Cloudflare: Build for the agent era",
    "cookie": "banner"
   },
   "llmsTxt": true,
   "markdown": true,
   "rank_agent": 6
  },
  {
   "rank_popularity": 52,
   "domain": "stripe.com",
   "url": "https://www.stripe.com/",
   "category": "Finance",
   "tier": "Mid",
   "score": 90,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 35,
    "extras": 5
   },
   "notes": [
    [
     "cookie banner",
     "warn"
    ],
    [
     "llms.txt",
     "good"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Stripe | Plateforme et solution de paiement en ligne",
    "cookie": "banner"
   },
   "llmsTxt": true,
   "markdown": false,
   "rank_agent": 17
  },
  {
   "rank_popularity": 53,
   "domain": "notion.so",
   "url": "https://notion.so/",
   "category": "Productivity",
   "tier": "Mid",
   "score": 95,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 40,
    "extras": 5
   },
   "notes": [
    [
     "llms.txt",
     "good"
    ]
   ],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "The AI workspace that works for you. | Notion",
    "cookie": "none"
   },
   "llmsTxt": true,
   "markdown": false,
   "rank_agent": 7
  },
  {
   "rank_popularity": 54,
   "domain": "figma.com",
   "url": "https://www.figma.com/",
   "category": "Design",
   "tier": "Mid",
   "score": 84,
   "grade": "B",
   "points": {
    "robots": 24.0,
    "http": 20,
    "browser": 40,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks 4/6 training crawlers",
     "warn"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": false,
    "CCBot": false,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Figma: The collaborative canvas for design, code, and AI",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 32
  },
  {
   "rank_popularity": 55,
   "domain": "producthunt.com",
   "url": "https://www.producthunt.com/",
   "category": "Tech",
   "tier": "Mid",
   "score": 55,
   "grade": "C",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 0,
    "extras": 5
   },
   "notes": [
    [
     "browser: Cloudflare challenge",
     "bad"
    ],
    [
     "llms.txt",
     "good"
    ]
   ],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "Just a moment...",
    "cookie": "none"
   },
   "llmsTxt": true,
   "markdown": false,
   "rank_agent": 56
  },
  {
   "rank_popularity": 56,
   "domain": "news.ycombinator.com",
   "url": "https://news.ycombinator.com/",
   "category": "Community",
   "tier": "Mid",
   "score": 90,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 40,
    "extras": 0
   },
   "notes": [],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Hacker News",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 18
  },
  {
   "rank_popularity": 57,
   "domain": "substack.com",
   "url": "https://www.substack.com/",
   "category": "Publishing",
   "tier": "Mid",
   "score": 90,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 35,
    "extras": 5
   },
   "notes": [
    [
     "cookie banner",
     "warn"
    ],
    [
     "llms.txt",
     "good"
    ]
   ],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Substack",
    "cookie": "banner"
   },
   "llmsTxt": true,
   "markdown": false,
   "rank_agent": 19
  },
  {
   "rank_popularity": 58,
   "domain": "coinbase.com",
   "url": "https://www.coinbase.com/",
   "category": "Crypto",
   "tier": "Mid",
   "score": 35,
   "grade": "D",
   "points": {
    "robots": 30.0,
    "http": 0,
    "browser": 0,
    "extras": 5
   },
   "notes": [
    [
     "agent fetch: Cloudflare challenge",
     "bad"
    ],
    [
     "browser: Cloudflare challenge",
     "bad"
    ],
    [
     "llms.txt",
     "good"
    ]
   ],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 403,
    "ok": false
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "Just a moment...",
    "cookie": "none"
   },
   "llmsTxt": true,
   "markdown": false,
   "rank_agent": 66
  },
  {
   "rank_popularity": 59,
   "domain": "binance.com",
   "url": "https://www.binance.com/",
   "category": "Crypto",
   "tier": "Mid",
   "score": 90,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 40,
    "extras": 0
   },
   "notes": [],
   "vendors": [
    "Akamai"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 202,
    "ok": true,
    "title": "Binance Crypto Exchange: Buy, Sell Bitcoin, Ethereum, Stocks & More",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 20
  },
  {
   "rank_popularity": 60,
   "domain": "coingecko.com",
   "url": "https://www.coingecko.com/",
   "category": "Crypto",
   "tier": "Mid",
   "score": 55,
   "grade": "C",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 0,
    "extras": 5
   },
   "notes": [
    [
     "browser: Cloudflare challenge",
     "bad"
    ],
    [
     "serves Markdown",
     "good"
    ]
   ],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "Just a moment...",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": true,
   "rank_agent": 57
  },
  {
   "rank_popularity": 61,
   "domain": "coinmarketcap.com",
   "url": "https://www.coinmarketcap.com/",
   "category": "Crypto",
   "tier": "Mid",
   "score": 95,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 40,
    "extras": 5
   },
   "notes": [
    [
     "llms.txt",
     "good"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Cryptocurrency Prices, Charts And Market Capitalizations | CoinMarketCap",
    "cookie": "none"
   },
   "llmsTxt": true,
   "markdown": false,
   "rank_agent": 8
  },
  {
   "rank_popularity": 62,
   "domain": "glassdoor.com",
   "url": "https://www.glassdoor.com/",
   "category": "Jobs",
   "tier": "Mid",
   "score": 16,
   "grade": "F",
   "points": {
    "robots": 15.5,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks user-driven agents",
     "bad"
    ],
    [
     "robots: blocks AI search",
     "bad"
    ],
    [
     "robots: blocks 5/6 training crawlers",
     "warn"
    ],
    [
     "agent fetch: Cloudflare challenge",
     "bad"
    ],
    [
     "browser: Cloudflare challenge",
     "bad"
    ]
   ],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": false,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": false,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": false,
    "CCBot": false,
    "Applebot-Extended": false,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 403,
    "ok": false
   },
   "browser": {
    "status": 401,
    "ok": false,
    "title": "Just a moment...",
    "cookie": "banner"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 90
  },
  {
   "rank_popularity": 63,
   "domain": "yelp.com",
   "url": "https://www.yelp.com/",
   "category": "Local",
   "tier": "Mid",
   "score": 30,
   "grade": "D",
   "points": {
    "robots": 30.0,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "agent fetch: HTTP 403",
     "bad"
    ],
    [
     "browser: HTTP 403",
     "bad"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": false,
   "http": {
    "status": 403,
    "ok": false
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 74
  },
  {
   "rank_popularity": 64,
   "domain": "ticketmaster.com",
   "url": "https://www.ticketmaster.com/",
   "category": "Tickets",
   "tier": "Mid",
   "score": 30,
   "grade": "D",
   "points": {
    "robots": 30.0,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "agent fetch: HTTP 403",
     "bad"
    ],
    [
     "browser: Kasada challenge",
     "bad"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": false,
   "http": {
    "status": 403,
    "ok": false
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "Your Browsing Activity Has Been Paused",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 75
  },
  {
   "rank_popularity": 65,
   "domain": "expedia.com",
   "url": "https://www.expedia.com/",
   "category": "Travel",
   "tier": "Mid",
   "score": 35,
   "grade": "D",
   "points": {
    "robots": 30.0,
    "http": 0,
    "browser": 0,
    "extras": 5
   },
   "notes": [
    [
     "agent fetch: Block page: Bot or Not?",
     "bad"
    ],
    [
     "browser: DataDome CAPTCHA, CAPTCHA iframe",
     "bad"
    ],
    [
     "llms.txt",
     "good"
    ]
   ],
   "vendors": [
    "Akamai"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 429,
    "ok": false
   },
   "browser": {
    "status": 429,
    "ok": false,
    "title": "Bot or Not?",
    "cookie": "none"
   },
   "llmsTxt": true,
   "markdown": false,
   "rank_agent": 67
  },
  {
   "rank_popularity": 66,
   "domain": "kayak.com",
   "url": "https://www.kayak.com/",
   "category": "Travel",
   "tier": "Mid",
   "score": 85,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 30,
    "extras": 5
   },
   "notes": [
    [
     "cookie wall blocks the page",
     "bad"
    ],
    [
     "llms.txt",
     "good"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Search Flights, Hotels & Rental Cars | KAYAK",
    "cookie": "wall"
   },
   "llmsTxt": true,
   "markdown": false,
   "rank_agent": 30
  },
  {
   "rank_popularity": 67,
   "domain": "wsj.com",
   "url": "https://www.wsj.com/",
   "category": "News",
   "tier": "Mid",
   "score": 10,
   "grade": "F",
   "points": {
    "robots": 10.0,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks user-driven agents",
     "bad"
    ],
    [
     "robots: blocks AI search",
     "bad"
    ],
    [
     "robots: blocks 4/6 training crawlers",
     "warn"
    ],
    [
     "agent fetch: DataDome CAPTCHA",
     "bad"
    ],
    [
     "browser: DataDome CAPTCHA, CAPTCHA iframe",
     "bad"
    ]
   ],
   "vendors": [
    "DataDome"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": false,
    "Perplexity-User": false,
    "OAI-SearchBot": true,
    "Claude-SearchBot": false,
    "PerplexityBot": false,
    "GPTBot": true,
    "ClaudeBot": false,
    "Google-Extended": false,
    "CCBot": false,
    "Applebot-Extended": true,
    "meta-externalagent": false
   },
   "robotsFound": true,
   "http": {
    "status": 401,
    "ok": false
   },
   "browser": {
    "status": 401,
    "ok": false,
    "title": "wsj.com",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 91
  },
  {
   "rank_popularity": 68,
   "domain": "bloomberg.com",
   "url": "https://www.bloomberg.com/",
   "category": "News",
   "tier": "Mid",
   "score": 23,
   "grade": "F",
   "points": {
    "robots": 18,
    "http": 0,
    "browser": 0,
    "extras": 5
   },
   "notes": [
    [
     "robots: blocks AI search",
     "bad"
    ],
    [
     "robots: blocks 6/6 training crawlers",
     "warn"
    ],
    [
     "agent fetch: HUMAN/PerimeterX CAPTCHA, CAPTCHA wall",
     "bad"
    ],
    [
     "browser: block page \"Bloomberg - Are you a robot?\"",
     "bad"
    ],
    [
     "llms.txt",
     "good"
    ]
   ],
   "vendors": [
    "HUMAN/PerimeterX"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": false,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": false,
    "CCBot": false,
    "Applebot-Extended": false,
    "meta-externalagent": false
   },
   "robotsFound": true,
   "http": {
    "status": 403,
    "ok": false
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "Bloomberg - Are you a robot?",
    "cookie": "none"
   },
   "llmsTxt": true,
   "markdown": false,
   "rank_agent": 84
  },
  {
   "rank_popularity": 69,
   "domain": "reuters.com",
   "url": "https://reuters.com/",
   "category": "News",
   "tier": "Mid",
   "score": 7,
   "grade": "F",
   "points": {
    "robots": 7,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks user-driven agents",
     "bad"
    ],
    [
     "robots: blocks AI search",
     "bad"
    ],
    [
     "robots: blocks 6/6 training crawlers",
     "warn"
    ],
    [
     "agent fetch: DataDome CAPTCHA",
     "bad"
    ],
    [
     "browser: DataDome CAPTCHA, CAPTCHA iframe",
     "bad"
    ]
   ],
   "vendors": [
    "DataDome"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": false,
    "Perplexity-User": false,
    "OAI-SearchBot": true,
    "Claude-SearchBot": false,
    "PerplexityBot": false,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": false,
    "CCBot": false,
    "Applebot-Extended": false,
    "meta-externalagent": false
   },
   "robotsFound": true,
   "http": {
    "status": 401,
    "ok": false
   },
   "browser": {
    "status": 401,
    "ok": false,
    "title": "reuters.com",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 94
  },
  {
   "rank_popularity": 70,
   "domain": "forbes.com",
   "url": "https://www.forbes.com/",
   "category": "News",
   "tier": "Mid",
   "score": 20,
   "grade": "F",
   "points": {
    "robots": 19.5,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks AI search",
     "bad"
    ],
    [
     "robots: blocks 5/6 training crawlers",
     "warn"
    ],
    [
     "agent fetch: DataDome CAPTCHA",
     "bad"
    ],
    [
     "browser: DataDome CAPTCHA, CAPTCHA iframe",
     "bad"
    ]
   ],
   "vendors": [
    "DataDome"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": false,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": true,
    "CCBot": false,
    "Applebot-Extended": false,
    "meta-externalagent": false
   },
   "robotsFound": true,
   "http": {
    "status": 403,
    "ok": false
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "forbes.com",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 89
  },
  {
   "rank_popularity": 71,
   "domain": "theverge.com",
   "url": "https://www.theverge.com/",
   "category": "News",
   "tier": "Mid",
   "score": 41,
   "grade": "D",
   "points": {
    "robots": 6.0,
    "http": 0,
    "browser": 35,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks user-driven agents",
     "bad"
    ],
    [
     "robots: blocks AI search",
     "bad"
    ],
    [
     "robots: blocks 4/6 training crawlers",
     "warn"
    ],
    [
     "agent fetch: HTTP 403",
     "bad"
    ],
    [
     "cookie banner",
     "warn"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": false,
    "Claude-User": false,
    "Perplexity-User": false,
    "OAI-SearchBot": true,
    "Claude-SearchBot": false,
    "PerplexityBot": false,
    "GPTBot": true,
    "ClaudeBot": false,
    "Google-Extended": false,
    "CCBot": false,
    "Applebot-Extended": true,
    "meta-externalagent": false
   },
   "robotsFound": true,
   "http": {
    "status": 403,
    "ok": false
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "The Verge",
    "cookie": "banner"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 64
  },
  {
   "rank_popularity": 72,
   "domain": "techcrunch.com",
   "url": "https://www.techcrunch.com/",
   "category": "News",
   "tier": "Mid",
   "score": 78,
   "grade": "B",
   "points": {
    "robots": 18.5,
    "http": 20,
    "browser": 40,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks user-driven agents",
     "bad"
    ],
    [
     "robots: blocks 5/6 training crawlers",
     "warn"
    ]
   ],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": false,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": false,
    "CCBot": false,
    "Applebot-Extended": false,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "TechCrunch | Startup and Technology News",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 37
  },
  {
   "rank_popularity": 73,
   "domain": "arstechnica.com",
   "url": "https://www.arstechnica.com/",
   "category": "News",
   "tier": "Mid",
   "score": 64,
   "grade": "C",
   "points": {
    "robots": 8.5,
    "http": 20,
    "browser": 35,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks user-driven agents",
     "bad"
    ],
    [
     "robots: blocks AI search",
     "bad"
    ],
    [
     "robots: blocks 5/6 training crawlers",
     "warn"
    ],
    [
     "cookie banner",
     "warn"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": false,
    "Perplexity-User": false,
    "OAI-SearchBot": true,
    "Claude-SearchBot": false,
    "PerplexityBot": false,
    "GPTBot": true,
    "ClaudeBot": false,
    "Google-Extended": false,
    "CCBot": false,
    "Applebot-Extended": false,
    "meta-externalagent": false
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Ars Technica - Serving the Technologist since 1998. News, reviews, and analysis.",
    "cookie": "banner"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 48
  },
  {
   "rank_popularity": 74,
   "domain": "npmjs.com",
   "url": "https://npmjs.com/",
   "category": "Developer",
   "tier": "Mid",
   "score": 30,
   "grade": "D",
   "points": {
    "robots": 30.0,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "agent fetch: Cloudflare challenge",
     "bad"
    ],
    [
     "browser: Cloudflare challenge",
     "bad"
    ]
   ],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": false,
   "http": {
    "status": 403,
    "ok": false
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "Just a moment...",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 76
  },
  {
   "rank_popularity": 75,
   "domain": "pypi.org",
   "url": "https://pypi.org/",
   "category": "Developer",
   "tier": "Mid",
   "score": 90,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 40,
    "extras": 0
   },
   "notes": [],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "PyPI \u00b7 The Python Package Index",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 21
  },
  {
   "rank_popularity": 76,
   "domain": "arxiv.org",
   "url": "https://arxiv.org/",
   "category": "Research",
   "tier": "Niche",
   "score": 90,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 40,
    "extras": 0
   },
   "notes": [],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "arXiv.org e-Print archive",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 22
  },
  {
   "rank_popularity": 77,
   "domain": "archive.org",
   "url": "https://archive.org/",
   "category": "Reference",
   "tier": "Niche",
   "score": 70,
   "grade": "B",
   "points": {
    "robots": 30.0,
    "http": 0,
    "browser": 40,
    "extras": 0
   },
   "notes": [
    [
     "agent fetch: JavaScript-only shell",
     "warn"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": false
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Internet Archive: Digital Library of Free & Borrowable Texts, Movies, Music & Wayback Machine",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 46
  },
  {
   "rank_popularity": 78,
   "domain": "huggingface.co",
   "url": "https://huggingface.co/",
   "category": "AI",
   "tier": "Niche",
   "score": 90,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 40,
    "extras": 0
   },
   "notes": [],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Hugging Face \u2013 The AI community building the future.",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 23
  },
  {
   "rank_popularity": 79,
   "domain": "lobste.rs",
   "url": "https://lobste.rs/",
   "category": "Community",
   "tier": "Niche",
   "score": 62,
   "grade": "C",
   "points": {
    "robots": 1.5,
    "http": 20,
    "browser": 40,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks user-driven agents",
     "bad"
    ],
    [
     "robots: blocks AI search",
     "bad"
    ],
    [
     "robots: blocks 5/6 training crawlers",
     "warn"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": false,
    "Claude-User": false,
    "Perplexity-User": false,
    "OAI-SearchBot": false,
    "Claude-SearchBot": false,
    "PerplexityBot": false,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": false,
    "CCBot": false,
    "Applebot-Extended": true,
    "meta-externalagent": false
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Lobsters",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 49
  },
  {
   "rank_popularity": 80,
   "domain": "dev.to",
   "url": "https://dev.to/",
   "category": "Developer",
   "tier": "Niche",
   "score": 95,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 40,
    "extras": 5
   },
   "notes": [
    [
     "llms.txt",
     "good"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "DEV Community",
    "cookie": "none"
   },
   "llmsTxt": true,
   "markdown": false,
   "rank_agent": 9
  },
  {
   "rank_popularity": 81,
   "domain": "docs.python.org",
   "url": "https://docs.python.org/",
   "category": "Developer",
   "tier": "Niche",
   "score": 90,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 40,
    "extras": 0
   },
   "notes": [],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Python 3.15.2 documentation",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 24
  },
  {
   "rank_popularity": 82,
   "domain": "developer.mozilla.org",
   "url": "https://developer.mozilla.org/",
   "category": "Developer",
   "tier": "Niche",
   "score": 90,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 40,
    "extras": 0
   },
   "notes": [],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "MDN Web Docs",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 25
  },
  {
   "rank_popularity": 83,
   "domain": "letterboxd.com",
   "url": "https://www.letterboxd.com/",
   "category": "Entertainment",
   "tier": "Niche",
   "score": 21,
   "grade": "F",
   "points": {
    "robots": 21,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks 6/6 training crawlers",
     "warn"
    ],
    [
     "agent fetch: Cloudflare challenge",
     "bad"
    ],
    [
     "browser: Cloudflare challenge",
     "bad"
    ]
   ],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": false,
    "CCBot": false,
    "Applebot-Extended": false,
    "meta-externalagent": false
   },
   "robotsFound": true,
   "http": {
    "status": 403,
    "ok": false
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "Just a moment...",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 88
  },
  {
   "rank_popularity": 84,
   "domain": "goodreads.com",
   "url": "https://www.goodreads.com/",
   "category": "Books",
   "tier": "Niche",
   "score": 47,
   "grade": "D",
   "points": {
    "robots": 27.0,
    "http": 20,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks 2/6 training crawlers",
     "warn"
    ],
    [
     "browser: AWS WAF challenge (202)",
     "bad"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": false,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": false,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 202,
    "ok": false,
    "title": "",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 60
  },
  {
   "rank_popularity": 85,
   "domain": "bandcamp.com",
   "url": "https://www.bandcamp.com/",
   "category": "Music",
   "tier": "Niche",
   "score": 22,
   "grade": "F",
   "points": {
    "robots": 22.5,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks 5/6 training crawlers",
     "warn"
    ],
    [
     "agent fetch: Block page: Client Challenge",
     "bad"
    ],
    [
     "browser: Block page: Client Challenge",
     "bad"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": false,
    "CCBot": false,
    "Applebot-Extended": true,
    "meta-externalagent": false
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": false
   },
   "browser": {
    "status": 200,
    "ok": false,
    "title": "Client Challenge",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 86
  },
  {
   "rank_popularity": 86,
   "domain": "discogs.com",
   "url": "https://www.discogs.com/",
   "category": "Music",
   "tier": "Niche",
   "score": 30,
   "grade": "D",
   "points": {
    "robots": 30.0,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "agent fetch: Cloudflare challenge",
     "bad"
    ],
    [
     "browser: Cloudflare challenge",
     "bad"
    ]
   ],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": false,
   "http": {
    "status": 403,
    "ok": false
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "Just a moment...",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 77
  },
  {
   "rank_popularity": 87,
   "domain": "allrecipes.com",
   "url": "https://www.allrecipes.com/",
   "category": "Food",
   "tier": "Niche",
   "score": 8,
   "grade": "F",
   "points": {
    "robots": 8.5,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks user-driven agents",
     "bad"
    ],
    [
     "robots: blocks AI search",
     "bad"
    ],
    [
     "robots: blocks 5/6 training crawlers",
     "warn"
    ],
    [
     "agent fetch: HTTP 402",
     "bad"
    ],
    [
     "browser: Cloudflare challenge",
     "bad"
    ]
   ],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": false,
    "Perplexity-User": false,
    "OAI-SearchBot": true,
    "Claude-SearchBot": false,
    "PerplexityBot": false,
    "GPTBot": true,
    "ClaudeBot": false,
    "Google-Extended": false,
    "CCBot": false,
    "Applebot-Extended": false,
    "meta-externalagent": false
   },
   "robotsFound": true,
   "http": {
    "status": 402,
    "ok": false
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "Just a moment...",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 93
  },
  {
   "rank_popularity": 88,
   "domain": "wikihow.com",
   "url": "https://www.wikihow.com/",
   "category": "Reference",
   "tier": "Niche",
   "score": 26,
   "grade": "F",
   "points": {
    "robots": 5.5,
    "http": 20,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks user-driven agents",
     "bad"
    ],
    [
     "robots: blocks AI search",
     "bad"
    ],
    [
     "robots: blocks 5/6 training crawlers",
     "warn"
    ],
    [
     "browser: Block page: Client Challenge",
     "bad"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": false,
    "Claude-User": false,
    "Perplexity-User": true,
    "OAI-SearchBot": false,
    "Claude-SearchBot": false,
    "PerplexityBot": false,
    "GPTBot": false,
    "ClaudeBot": false,
    "Google-Extended": true,
    "CCBot": false,
    "Applebot-Extended": false,
    "meta-externalagent": false
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": false,
    "title": "Client Challenge",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 82
  },
  {
   "rank_popularity": 89,
   "domain": "genius.com",
   "url": "https://www.genius.com/",
   "category": "Music",
   "tier": "Niche",
   "score": 30,
   "grade": "D",
   "points": {
    "robots": 30.0,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "agent fetch: Cloudflare challenge",
     "bad"
    ],
    [
     "browser: Cloudflare challenge",
     "bad"
    ]
   ],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": false,
   "http": {
    "status": 403,
    "ok": false
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "Just a moment...",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 78
  },
  {
   "rank_popularity": 90,
   "domain": "rottentomatoes.com",
   "url": "https://www.rottentomatoes.com/",
   "category": "Entertainment",
   "tier": "Niche",
   "score": 80,
   "grade": "B",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 30,
    "extras": 0
   },
   "notes": [
    [
     "cookie wall blocks the page",
     "bad"
    ]
   ],
   "vendors": [
    "Akamai"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Rotten Tomatoes: Movies | TV Shows | Movie Trailers | Reviews | Rotten Tomatoes",
    "cookie": "wall"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 34
  },
  {
   "rank_popularity": 91,
   "domain": "metacritic.com",
   "url": "https://www.metacritic.com/",
   "category": "Entertainment",
   "tier": "Niche",
   "score": 78,
   "grade": "B",
   "points": {
    "robots": 22.5,
    "http": 20,
    "browser": 35,
    "extras": 0
   },
   "notes": [
    [
     "robots: blocks AI search",
     "bad"
    ],
    [
     "robots: blocks 3/6 training crawlers",
     "warn"
    ],
    [
     "cookie banner",
     "warn"
    ]
   ],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": false,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": false,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": false,
    "Applebot-Extended": true,
    "meta-externalagent": false
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Movie Reviews, TV Reviews, Game Reviews, and Music Reviews - Metacritic",
    "cookie": "banner"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 38
  },
  {
   "rank_popularity": 92,
   "domain": "duckduckgo.com",
   "url": "https://duckduckgo.com/",
   "category": "Search",
   "tier": "Niche",
   "score": 90,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 40,
    "extras": 0
   },
   "notes": [],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "DuckDuckGo - Protection. Privacy. Peace of mind.",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 26
  },
  {
   "rank_popularity": 93,
   "domain": "openstreetmap.org",
   "url": "https://openstreetmap.org/",
   "category": "Maps",
   "tier": "Niche",
   "score": 90,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 40,
    "extras": 0
   },
   "notes": [],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "OpenStreetMap",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 27
  },
  {
   "rank_popularity": 94,
   "domain": "gutenberg.org",
   "url": "https://gutenberg.org/",
   "category": "Books",
   "tier": "Niche",
   "score": 90,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 40,
    "extras": 0
   },
   "notes": [],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Free eBooks | Project Gutenberg",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 28
  },
  {
   "rank_popularity": 95,
   "domain": "lichess.org",
   "url": "https://lichess.org/",
   "category": "Games",
   "tier": "Niche",
   "score": 95,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 40,
    "extras": 5
   },
   "notes": [
    [
     "serves Markdown",
     "good"
    ]
   ],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "lichess.org \u2022 Free Online Chess",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": true,
   "rank_agent": 10
  },
  {
   "rank_popularity": 96,
   "domain": "polymarket.com",
   "url": "https://polymarket.com/",
   "category": "Prediction markets",
   "tier": "Niche",
   "score": 100,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 40,
    "extras": 10
   },
   "notes": [
    [
     "llms.txt",
     "good"
    ],
    [
     "serves Markdown",
     "good"
    ]
   ],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Polymarket | The World\u2019s Largest Prediction Market\u2122",
    "cookie": "none"
   },
   "llmsTxt": true,
   "markdown": true,
   "rank_agent": 1
  },
  {
   "rank_popularity": 97,
   "domain": "defillama.com",
   "url": "https://defillama.com/",
   "category": "Crypto",
   "tier": "Niche",
   "score": 35,
   "grade": "D",
   "points": {
    "robots": 30.0,
    "http": 0,
    "browser": 0,
    "extras": 5
   },
   "notes": [
    [
     "agent fetch: Cloudflare challenge",
     "bad"
    ],
    [
     "browser: Cloudflare challenge",
     "bad"
    ],
    [
     "llms.txt",
     "good"
    ]
   ],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 403,
    "ok": false
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "Just a moment...",
    "cookie": "none"
   },
   "llmsTxt": true,
   "markdown": false,
   "rank_agent": 68
  },
  {
   "rank_popularity": 98,
   "domain": "etherscan.io",
   "url": "https://etherscan.io/",
   "category": "Crypto",
   "tier": "Niche",
   "score": 60,
   "grade": "C",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 0,
    "extras": 10
   },
   "notes": [
    [
     "browser: Cloudflare challenge",
     "bad"
    ],
    [
     "llms.txt",
     "good"
    ],
    [
     "serves Markdown",
     "good"
    ]
   ],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "Just a moment...",
    "cookie": "none"
   },
   "llmsTxt": true,
   "markdown": true,
   "rank_agent": 54
  },
  {
   "rank_popularity": 99,
   "domain": "dexscreener.com",
   "url": "https://dexscreener.com/",
   "category": "Crypto",
   "tier": "Niche",
   "score": 30,
   "grade": "D",
   "points": {
    "robots": 30.0,
    "http": 0,
    "browser": 0,
    "extras": 0
   },
   "notes": [
    [
     "agent fetch: Cloudflare challenge",
     "bad"
    ],
    [
     "browser: Cloudflare challenge",
     "bad"
    ]
   ],
   "vendors": [
    "Cloudflare"
   ],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": false,
   "http": {
    "status": 403,
    "ok": false
   },
   "browser": {
    "status": 403,
    "ok": false,
    "title": "Just a moment...",
    "cookie": "none"
   },
   "llmsTxt": false,
   "markdown": false,
   "rank_agent": 79
  },
  {
   "rank_popularity": 100,
   "domain": "context.dev",
   "url": "https://context.dev/",
   "category": "AI",
   "tier": "Niche",
   "score": 100,
   "grade": "A",
   "points": {
    "robots": 30.0,
    "http": 20,
    "browser": 40,
    "extras": 10
   },
   "notes": [
    [
     "llms.txt",
     "good"
    ],
    [
     "serves Markdown",
     "good"
    ]
   ],
   "vendors": [],
   "robots": {
    "ChatGPT-User": true,
    "Claude-User": true,
    "Perplexity-User": true,
    "OAI-SearchBot": true,
    "Claude-SearchBot": true,
    "PerplexityBot": true,
    "GPTBot": true,
    "ClaudeBot": true,
    "Google-Extended": true,
    "CCBot": true,
    "Applebot-Extended": true,
    "meta-externalagent": true
   },
   "robotsFound": true,
   "http": {
    "status": 200,
    "ok": true
   },
   "browser": {
    "status": 200,
    "ok": true,
    "title": "Context.dev: Web Scraping API for AI Agents (Scrape, Search, Crawl)",
    "cookie": "none"
   },
   "llmsTxt": true,
   "markdown": true,
   "rank_agent": 2
  }
 ]
}