Бесплатный парсер парсера Shodan для поиска целей

curl 'https://www.shodan.io/search?query=FORTINET' \
  -H 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8' \
  -H 'accept-language: ru-RU,ru;q=0.5' \
  -H 'cache-control: max-age=0' \
  -b 'cf_clearance=zTwU1StnLnzduThbl4Sf5D5Opk2bSkqwx4igTRsckK4-1779574426-1.2.1.1-we7f4OVKLHphex_cASN9sZS5EmCeB1J62iIWi1vnfUGWsJ1w7G0oS0vh6_HgtrbwKBQ53IgDlzf6kvBgwz1sRP.Xws4czsCYtrfavB4Chk5aAdH2Icr1NdYLFu34BTZTtIfXMCBSnL6mDlaKAFbhZiugF7VMe4hQ90.Bi5fGQEgXCG7uRl1ZsMVqAcsrrNwN7UIVD3K1fTXVW8iMKz3P5zBpdUExtX9_odRxgNXr5nWxLZua6iDbosK3pGetA_QKmlql5k3fFoF9M3ymuWSVb7Ik7YPR69MpgUtQvXa5D4aMNgLBRW.X1cQ7yCghr.lBfHFF30kQzcZMKMgwQYuB8cvAyDkvx5qs42Hi2PLfB7gCTnVM_JurZ_Ge7ZLaw7iieqqpRdnM5_IUN9vU0cN7IKlgy3zEaHBXCxRPZkbDyTc' \
  -H 'priority: u=0, i' \
  -H 'referer: https://www.shodan.io/' \
  -H 'sec-ch-ua: "Chromium";v="148", "Brave";v="148", "Not/A)Brand";v="99"' \
  -H 'sec-ch-ua-arch: "x86"' \
  -H 'sec-ch-ua-bitness: "64"' \
  -H 'sec-ch-ua-full-version-list: "Chromium";v="148.0.0.0", "Brave";v="148.0.0.0", "Not/A)Brand";v="99.0.0.0"' \
  -H 'sec-ch-ua-mobile: ?0' \
  -H 'sec-ch-ua-model: ""' \
  -H 'sec-ch-ua-platform: "Windows"' \
  -H 'sec-ch-ua-platform-version: "19.0.0"' \
  -H 'sec-fetch-dest: document' \
  -H 'sec-fetch-mode: navigate' \
  -H 'sec-fetch-site: same-origin' \
  -H 'sec-fetch-user: ?1' \
  -H 'sec-gpc: 1' \
  -H 'upgrade-insecure-requests: 1' \
  -H 'user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36'

# Ждём появления результатов или Cloudflare challenge
deadline = time.time() + 20
while time.time() < deadline:
    html = driver.page_source
    if 'class="result"' in html:
        break  # результаты загрузились
    if "just a moment" in html.lower():
        time.sleep(2)  # ждём пока Cloudflare отпустит
        continue
    time.sleep(1)

sleep_sec = interval + random.uniform(-jitter, jitter)
# interval=30, jitter=5 → пауза от 25 до 35 секунд

# Собрать образ
docker compose build
# Запустить в фоне
docker compose up -d
# Логи
docker compose logs -f
# Остановить
docker compose down

command:
  - --query
  - FORTINET      # поисковый запрос
  - --interval
  - "30"          # секунд между раундами
  - --pages
  - "3"           # страниц за раунд

--query      Поисковый запрос (default: FORTINET)
--interval   Секунд между раундами (default: 30)
--pages      Страниц за один раунд (default: 1)
--rounds     Количество раундов, 0 = бесконечно (default: 0)
--output     Путь к файлу БД (default: shodan_results.db)
--headless   Без окна браузера
--export-csv Только экспорт БД в CSV, без парсинга

undetected-chromedriver==3.5.5
selenium>=4.0.0
beautifulsoup4>=4.12.0
setuptools
packaging

#!/usr/bin/env python3
"""
Shodan scraper через живой браузер (undetected-chromedriver).
Cloudflare обходится автоматически — никаких cookie вручную.

Установка:
    pip install undetected-chromedriver selenium beautifulsoup4

Использование:
    python shodan_scraper.py --query FORTINET --interval 30
    python shodan_scraper.py --query FORTINET --interval 30 --headless   # без окна
    python shodan_scraper.py --query FORTINET --rounds 5 --pages 3
"""

import argparse
import csv
import re
import sqlite3
import time
import random
from datetime import datetime
from pathlib import Path

from bs4 import BeautifulSoup

# ── ANSI ──────────────────────────────────────────────────────────────────────
R="\033[0m"; BOLD="\033[1m"; DIM="\033[2m"
RED="\033[31m"; GREEN="\033[32m"; YELLOW="\033[33m"; CYAN="\033[36m"; WHITE="\033[97m"
def c(t,*cc): return "".join(cc)+str(t)+R
def hr(ch="─",w=90,col=DIM): print(c(ch*w,col))

# ── Terminal output ────────────────────────────────────────────────────────────
def print_header(query,interval,db_path,headless):
    hr("═",col=CYAN)
    mode = "HEADLESS" if headless else "VISIBLE BROWSER"
    print(c("  SHODAN SCRAPER",BOLD,CYAN)+c(f"  [{mode}]  query={query!r}  interval={interval}s  db={db_path}",DIM))
    hr("═",col=CYAN)

def print_request(rnd,page,url):
    print()
    hr()
    print(c(f"  ▶  ROUND {rnd}  PAGE {page}",BOLD,YELLOW))
    print(c("     URL : ",DIM)+c(url,WHITE))

def print_response(nbytes,ms,title=None):
    line=c(f"     OK  {nbytes} bytes  {ms:.0f} ms",GREEN,BOLD)
    if title:
        line += c(f"  <title>{title[:60]}</title>",DIM)
    print(line)

def print_table(records,new_flags):
    COL=[18,6,22,36]
    HDR=["IP","PORT","COUNTRY","ORGANIZATION"]
    sep="┼".join("─"*(w+2) for w in COL)
    rfmt="│".join(f" {{:<{w}}} " for w in COL)
    print()
    print(c("  "+"┬".join("─"*(w+2) for w in COL),DIM))
    print(c("  "+rfmt.format(*HDR),BOLD,CYAN))
    print(c("  "+sep,DIM))
    for rec,is_new in zip(records,new_flags):
        ip   = (rec["ip"]           or "")[:COL[0]]
        port = str(rec["port"]      or "")[:COL[1]]
        coun = (rec["country"]      or "")[:COL[2]]
        org  = (rec["organization"] or "")[:COL[3]]
        mark = c("  ★",GREEN,BOLD) if is_new else c("  ·",DIM)
        line = "  "+rfmt.format(ip,port,coun,org)
        print((c(line,GREEN) if is_new else line)+mark)
    print(c("  "+"┴".join("─"*(w+2) for w in COL),DIM))

def print_sleep(sec):
    print(c(f"\n  ⏱  следующий раунд через {sec:.1f} с  (Ctrl+C для остановки)\n",DIM))

# ── DB ────────────────────────────────────────────────────────────────────────
def init_db(path):
    conn=sqlite3.connect(path,check_same_thread=False)
    conn.execute("""CREATE TABLE IF NOT EXISTS hosts(
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        ip TEXT NOT NULL, port INTEGER, country TEXT,
        organization TEXT, hostname TEXT, seen_at TEXT NOT NULL,
        UNIQUE(ip,port))""")
    conn.commit()
    return conn

def upsert(conn,rec):
    cur=conn.execute(
        "INSERT OR IGNORE INTO hosts(ip,port,country,organization,hostname,seen_at)"
        " VALUES(:ip,:port,:country,:organization,:hostname,:seen_at)",rec)
    conn.commit()
    return cur.rowcount>0

def db_count(conn):
    return conn.execute("SELECT COUNT(*) FROM hosts").fetchone()[0]

def export_csv(conn,path):
    rows=conn.execute(
        "SELECT ip,port,country,organization,hostname,seen_at FROM hosts ORDER BY id"
    ).fetchall()
    with open(path,"w",newline="",encoding="utf-8") as f:
        w=csv.writer(f)
        w.writerow(["ip","port","country","organization","hostname","seen_at"])
        w.writerows(rows)
    print(c(f"\n  CSV → {path}  ({len(rows)} строк)",CYAN,BOLD))

# ── Parser ────────────────────────────────────────────────────────────────────
def get_page_title(html):
    m=re.search(r"<title>(.*?)</title>",html,re.I|re.S)
    return m.group(1).strip()[:80] if m else ""

def parse_results(html):
    soup=BeautifulSoup(html,"html.parser")
    out=[]
    for div in soup.select("div.result"):
        title_tag=div.select_one("a.title")
        if not title_tag: continue
        ip=title_tag.get_text(strip=True)

        port=None
        ext=div.select_one("a[rel~='noopener']")
        if ext and ext.get("href"):
            m=re.search(r":(\d+)/?$",ext["href"])
            if m: port=int(m.group(1))

        flag=div.select_one("img.flag")
        country=flag["title"] if flag and flag.get("title") else None

        org_t=div.select_one("a.filter-org")
        org=org_t.get_text(strip=True) if org_t else None

        host_t=div.select_one("li.hostnames")
        hostname=host_t.get_text(strip=True) if host_t else None

        out.append({"ip":ip,"port":port,"country":country,
                    "organization":org,"hostname":hostname,
                    "seen_at":datetime.utcnow().isoformat(timespec="seconds")})
    return out

# ── Browser ───────────────────────────────────────────────────────────────────
def find_chrome_binary():
    """Ищет Chrome или Chromium на системе."""
    import shutil
    candidates = [
        "google-chrome", "google-chrome-stable",
        "chromium", "chromium-browser",
        "/usr/bin/google-chrome", "/usr/bin/chromium",
        "/usr/bin/chromium-browser",
        "/opt/google/chrome/chrome",
    ]
    for name in candidates:
        path = shutil.which(name) or (name if Path(name).exists() else None)
        if path:
            return path
    return None

def create_driver(headless=False):
    """Создаёт undetected Chrome драйвер."""
    try:
        import undetected_chromedriver as uc
    except Exception as e:
        print(c(f"  ✖  Ошибка импорта undetected-chromedriver: {e}",RED,BOLD))
        print(c("      Попробуй: pip install --upgrade undetected-chromedriver selenium",YELLOW))
        raise SystemExit(1)

    # Ищем браузер
    chrome_bin = find_chrome_binary()
    if chrome_bin:
        print(c(f"  ✔  Найден браузер: {chrome_bin}",GREEN))
    else:
        print(c("  ✖  Chrome/Chromium не найден!",RED,BOLD))
        print(c("      Kali Linux: sudo apt install chromium",YELLOW))
        print(c("      Или: wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && sudo dpkg -i google-chrome*.deb",YELLOW))
        raise SystemExit(1)

    import os
    display = os.environ.get("DISPLAY", "")
    if not display and not headless:
        print(c("  ℹ  DISPLAY не найден → включаем headless режим",YELLOW))
        headless = True

    options = uc.ChromeOptions()
    options.binary_location = chrome_bin
    options.add_argument("--no-sandbox")
    options.add_argument("--disable-dev-shm-usage")
    options.add_argument("--window-size=1920,1080")
    options.add_argument("--lang=en-US")
    options.add_argument("--disable-blink-features=AutomationControlled")
    options.add_argument("--disable-gpu")
    options.add_argument("--remote-debugging-port=0")

    if headless:
        options.add_argument("--headless=new")
        options.add_argument("--disable-gpu")

    try:
        driver = uc.Chrome(options=options, version_main=147)
    except Exception as e:
        print(c(f"  ✖  Не удалось запустить Chrome: {e}",RED,BOLD))
        print(c("      Убедись что ChromeDriver совместим с версией Chrome",YELLOW))
        print(c("      Попробуй: pip install --upgrade undetected-chromedriver",YELLOW))
        raise SystemExit(1)

    driver.set_page_load_timeout(40)
    return driver

def fetch_page(driver, url, wait_sec=4):
    """
    Открывает URL в браузере, ждёт загрузки, возвращает (html, ms, title).
    """
    t0 = time.perf_counter()
    driver.get(url)

    # Ждём пока появятся результаты или пройдёт таймаут
    deadline = time.time() + 20
    while time.time() < deadline:
        html = driver.page_source
        if 'class="result"' in html:
            break
        # Cloudflare challenge — ждём дольше
        if "just a moment" in html.lower() or "cf-browser-verification" in html.lower():
            print(c("  ⏳ Cloudflare challenge, ждём...",YELLOW),end="\r")
            time.sleep(2)
            continue
        time.sleep(1)

    ms = (time.perf_counter()-t0)*1000
    html = driver.page_source
    title = driver.title
    return html, ms, title

# ── Main loop ─────────────────────────────────────────────────────────────────
def poll(query,interval,db_path,max_rounds,jitter,pages_per_round,headless):
    conn=init_db(db_path)
    total_new=0
    round_num=0

    print_header(query,interval,db_path,headless)
    print(c("  Запускаем браузер...",DIM))

    driver = create_driver(headless=headless)

    # Первый визит на главную — получаем cf_clearance автоматически
    print(c("  Открываем shodan.io для получения cookies...",DIM))
    try:
        driver.get("https://www.shodan.io/")
        time.sleep(3)  # даём Cloudflare обработать
    except Exception:
        pass

    print(c("  Браузер готов. Начинаем сбор.\n",GREEN,BOLD))

    try:
        while max_rounds==0 or round_num<max_rounds:
            round_num+=1

            for page in range(1,pages_per_round+1):
                url=f"https://www.shodan.io/search?query={query}&page={page}"
                print_request(round_num,page,url)

                try:
                    html,ms,title=fetch_page(driver,url)
                except Exception as e:
                    print(c(f"\n  ✖  Ошибка браузера: {e}",RED,BOLD))
                    break

                print_response(len(html.encode()),ms,title)

                # Проверяем блокировку
                if 'class="result"' not in html:
                    print(c(f"\n  ⚠  Результатов нет на странице. Title: {title}",YELLOW))
                    print(c(f"     Сохраняю HTML → debug_last.html",DIM))
                    Path("debug_last.html").write_text(html,encoding="utf-8")
                    break

                records=parse_results(html)
                if not records:
                    print(c(f"\n  ⚠  Парсер нашёл 0 записей",YELLOW))
                    break

                new_flags=[upsert(conn,r) for r in records]
                new_count=sum(new_flags)
                total_new+=new_count
                print_table(records,new_flags)
                print(c(f"\n  parsed={len(records)}  new={new_count}  DB total={db_count(conn)}",BOLD))

                # Пауза между страницами
                if page<pages_per_round:
                    pause=random.uniform(3.0,7.0)
                    print(c(f"  ⏳ пауза {pause:.1f}с между страницами...",DIM))
                    time.sleep(pause)

            # Пауза между раундами
            if max_rounds==0 or round_num<max_rounds:
                sleep_sec=max(1.0,interval+random.uniform(-jitter,jitter))
                print_sleep(sleep_sec)
                time.sleep(sleep_sec)

    except KeyboardInterrupt:
        print(c("\n\n  Остановлено.",YELLOW,BOLD))
    finally:
        try:
            driver.quit()
        except Exception:
            pass
        csv_path=(db_path[:-3]+".csv") if db_path.endswith(".db") else db_path+".csv"
        export_csv(conn,csv_path)
        print(c(f"  Всего новых уникальных хостов: {total_new}",BOLD,GREEN))
        conn.close()

# ── CLI ───────────────────────────────────────────────────────────────────────
def main():
    p=argparse.ArgumentParser(
        description="Shodan scraper через живой браузер (Cloudflare обходится автоматически)"
    )
    p.add_argument("--query",    default="FORTINET",   help="Поисковый запрос Shodan")
    p.add_argument("--interval", type=float,default=30,help="Секунд между раундами")
    p.add_argument("--output",   default="shodan_results.db")
    p.add_argument("--rounds",   type=int,  default=0, help="0 = бесконечно")
    p.add_argument("--jitter",   type=float,default=5.0)
    p.add_argument("--pages",    type=int,  default=1, help="Страниц за раунд")
    p.add_argument("--headless", action="store_true",  help="Без окна браузера")
    p.add_argument("--export-csv",action="store_true", help="Только экспорт БД в CSV")
    args=p.parse_args()

    if args.export_csv:
        conn=init_db(args.output)
        csv_path=(args.output[:-3]+".csv") if args.output.endswith(".db") else args.output+".csv"
        export_csv(conn,csv_path)
        conn.close()
        return

    poll(args.query,args.interval,args.output,
         args.rounds,args.jitter,args.pages,args.headless)

if __name__=="__main__":
    main()