Upload 16 files

.gitattributes

@@ -1,35 +1,71 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.sqlite filter=lfs diff=lfs merge =lfs -bin*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.sqlite filter=lfs diff=lfs merge =lfs -bin

.gitignore

@@ -0,0 +1,6 @@
+.venv/
+__pycache__/
+cache.sqlite
+db/cache.sqlite
+.cloudflare-headers-cache.json
+.vscode/settings.json

Dockerfile

@@ -0,0 +1,40 @@
+FROM python:3.10
+
+# Set environment to non-interactive (avoids tzdata prompts)
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Update packages and install dependencies for Rust
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    curl \
+    build-essential \
+    pkg-config \
+    libffi-dev \
+    libssl-dev \
+    ca-certificates \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Rust (via official rustup script)
+RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
+
+# Add Rust to PATH
+ENV PATH="/root/.cargo/bin:$PATH"
+
+# Set working directory
+WORKDIR /app
+
+# Copy your files
+COPY . /app
+ENV TMPDIR=/app/tmp
+RUN mkdir -p /app/tmp && chmod -R 777 /app/tmp
+RUN mkdir -p /app/.cache && chmod -R 777 /app/.cache
+RUN chmod -R 777 /app
+
+# Confirm versions
+RUN rustc --version && cargo --version
+
+# Optional: install Python deps
+RUN python3 -m pip install -r requirements.txt
+
+EXPOSE 7860
+CMD [ "python", "app.py" ]

README.md

@@ -1,11 +1,11 @@
----
-title: Infinite Craft Searcher
-emoji: 📊
-colorFrom: indigo
-colorTo: pink
-sdk: docker
-pinned: false
-license: mit
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: Infinite Craft Searcher
+emoji: 🌖
+colorFrom: blue
+colorTo: red
+sdk: docker
+pinned: false
+license: mit
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

api.py

@@ -0,0 +1,93 @@
+import time
+
+from curl_cffi import requests
+
+from models import Element, Pair, PendingPair
+
+
+def raw_make_pair(
+    first: str,
+    second: str,
+    headers: dict[str, str],
+    *,
+    timeout: float = 30,
+) -> tuple[str, str | None, bool | None]:
+    response = requests.Session(impersonate='chrome').get(
+        "https://neal.fun/api/infinite-craft/pair",
+        params={"first": first, "second": second},
+        headers=headers,
+        timeout=timeout * 1000,
+    )
+    response.raise_for_status()
+    data = response.json()
+
+    if "result" not in data:
+        msg = f"Invalid response: {data!r}"
+        raise ValueError(msg)
+
+    return data["result"], data.get("emoji"), data.get("isNew")
+
+
+def make_pair(
+    pair: PendingPair,
+    headers: dict[str, str],
+    *,
+    timeout: float = 30,
+) -> Pair:
+    result, emoji, is_new = raw_make_pair(
+        pair.first.name,
+        pair.second.name,
+        headers,
+        timeout=timeout,
+    )
+    return Pair(
+        pair.first,
+        pair.second,
+        Element(result, emoji),
+        is_new,
+    )
+
+
+def make_pair_exp_backoff(
+    pair: PendingPair,
+    headers: dict[str, str],
+    *,
+    timeout: float = 30,
+) -> Pair:
+    started_at = time.perf_counter()
+    backoff = 1
+    while True:
+        exc = None
+        try:
+            eta = timeout - (time.perf_counter() - started_at)
+            return make_pair(pair, headers, timeout=eta)
+        except requests.RequestsError as e:
+            if e.args and e.args[0].startswith("HTTP Error 500:"):
+                raise  # don't bother retrying
+            exc = e
+        except Exception as e:
+            exc = e
+
+        eta = timeout - (time.perf_counter() - started_at)
+        if eta < backoff:
+            msg = f"Ran out of time while making the pair: {pair}"
+            raise TimeoutError(msg) from exc
+
+        time.sleep(backoff)
+        backoff = min(backoff * 2, 60)
+
+
+if __name__ == "__main__":
+    import cloudflare
+
+    headers = cloudflare.get_headers()
+
+    pair = make_pair(
+        PendingPair(
+            Element(input("First Element: ")),
+            Element(input("Second Element: ")),
+        ),
+        headers,
+    )
+
+    print(pair)

app.py

@@ -0,0 +1,44 @@
+import os
+import threading
+import time
+from fastapi import FastAPI
+from fastapi.responses import FileResponse, HTMLResponse, PlainTextResponse, StreamingResponse
+from fastapi.staticfiles import StaticFiles
+import uvicorn
+
+from run import start_finder
+
+
+app = FastAPI()
+app.mount('/file/db', StaticFiles(directory='db/', html=False), '/file/db')
+def tail_file(path):
+    with open(path, "r", encoding="utf-8") as f:
+        f.seek(0, os.SEEK_END)  # start at EOF
+        while True:
+            line = f.readline()
+            if line:
+                yield f"data: {line.strip()}\n\n"
+            else:
+                time.sleep(1)
+                
+@app.get("/raw-pair")
+def send_raw_pair():
+    with open('public/pair.txt') as f:
+        return PlainTextResponse(f.read())
+
+@app.get("/live-pair")
+def live_pairs():
+    with open('public/live-pairs.html') as f:
+        return HTMLResponse(f.read())
+
+@app.get("/stream-pair")
+def stream_pairs():
+    return StreamingResponse(tail_file('public/pair.txt'), media_type="text/event-stream")
+
+@app.on_event('startup')
+def start():
+    runner = threading.Thread(target=start_finder, daemon=True)
+    runner.start()
+
+if __name__ == "__main__":
+    uvicorn.run(app, host='0.0.0.0', port=7860)

cloudflare.py

@@ -0,0 +1,87 @@
+import contextlib
+import json
+import re
+from pathlib import Path
+import time
+
+import api
+import misc
+
+
+def parse_curl_string(curl_string: str) -> dict[str, str]:
+    headers = {}
+    header_lines = curl_string.strip().splitlines()
+
+    for header_line in header_lines:
+        found = re.search(r"-H\s*'(.*?)\s*:\s*(.*)'", header_line)
+        if not found:
+            continue
+
+        headers[found.group(1)] = found.group(2)
+
+    return headers
+
+
+def prompt_for_headers() -> dict[str, str]:
+    while True:
+        print(
+            "Follow these directions to bypass CloudFlare anti-bot restrictions.\n"
+            "  1. Go to https://neal.fun/infinite-craft/\n"
+            "  2. Open the developer tools, and select the Network tab\n"
+            "  3. Make any pair\n"
+            "  4. In the Network tab, find the network request that was made (search 'pair')\n"
+            "  5. Right click on it -> Copy -> Copy as cURL (bash)\n"
+            "  6. Come back to the terminal, and press 'Enter'.",
+        )
+        curl_string = misc.header_bank_and_impersonate()
+        if curl_string.startswith("curl 'https://neal.fun/api/infinite-craft/pair?first="):
+            headers = parse_curl_string(curl_string)
+            if headers:
+                return headers
+
+        print(
+            "Hmm... it doesn't look like your clipboard contains the right data. Try again.\n\n",
+        )
+
+
+def verify_headers(headers: dict[str, str]) -> Exception | None:
+    try:
+        api.raw_make_pair("Fire", "Water", headers)
+    except Exception as e:
+        return e
+
+    return None
+
+
+def get_headers(verify: bool = True) -> dict[str, str]:
+    filename = Path(__file__).parent / ".cloudflare-headers-cache.json"
+    try:
+        with filename.open() as f:
+            headers = json.load(f)
+    except Exception as e:
+        pass
+    else:
+        if verify and verify_headers(headers) is None:
+            return headers
+        with contextlib.suppress(Exception):
+            filename.unlink()
+
+    headers = prompt_for_headers()
+    if verify:
+        error = verify_headers(headers)
+        if error is not None:
+            print(f"Those headers are not valid! Error: {error!r}")
+            print("Try again.\n")
+            time.sleep(90)
+            return get_headers(verify)
+
+    with contextlib.suppress(Exception), filename.open("w") as f:
+        json.dump(headers, f, indent=2)
+
+    return headers
+
+
+if __name__ == "__main__":
+    headers = get_headers()
+    print("Your (working) headers:")
+    print(json.dumps(headers, indent=2))

db/persistence.py

@@ -0,0 +1,192 @@
+import random
+import sqlite3
+from typing import Generator, Literal
+
+from models import Element, Pair, PendingPair
+
+
+def connect() -> sqlite3.Connection:
+    return sqlite3.connect("db/cache.sqlite")
+
+
+with connect() as conn:
+    conn.execute(
+        """
+        CREATE TABLE IF NOT EXISTS element (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            first_created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+            name TEXT UNIQUE,
+            emoji TEXT
+        )
+        """,
+    )
+
+    conn.execute(
+        """
+        CREATE TABLE IF NOT EXISTS pair (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
+            first_element_id INTEGER,
+            second_element_id INTEGER,
+            result_element_id INTEGER,
+            is_discovery INTEGER,
+            FOREIGN KEY (first_element_id) REFERENCES element (id),
+            FOREIGN KEY (second_element_id) REFERENCES element (id),
+            FOREIGN KEY (result_element_id) REFERENCES element (id)
+            UNIQUE(first_element_id, second_element_id)
+        )
+        """,
+    )
+
+
+def _upsert_element(conn: sqlite3.Connection, element: Element) -> None:
+    conn.execute(
+        """
+        INSERT INTO element (name, emoji)
+        VALUES (?, ?)
+        ON CONFLICT(name) DO UPDATE SET
+        emoji = excluded.emoji
+        """,
+        (element.name, element.emoji),
+    )
+
+    (element.database_id,) = conn.execute(
+        "SELECT id FROM element WHERE name = ?",
+        (element.name,),
+    ).fetchone()
+
+
+def _upsert_pair(conn: sqlite3.Connection, pair: Pair) -> None:
+    # first, insert the elements:
+    for element in pair.elements:
+        if element.database_id is not None:
+            continue
+
+        _upsert_element(conn, element)
+
+    # now, record the pair:
+    conn.execute(
+        """
+        INSERT INTO pair (first_element_id, second_element_id, result_element_id, is_discovery)
+        VALUES (?, ?, ?, ?)
+        ON CONFLICT(first_element_id, second_element_id) DO UPDATE SET
+        result_element_id = excluded.result_element_id,
+        is_discovery = MAX(is_discovery, excluded.is_discovery)
+        """,
+        (*(e.database_id for e in pair.elements), 1 if pair.is_discovery else 0),
+    )
+
+
+def record_pair(pair: Pair) -> None:
+    with connect() as conn:
+        _upsert_pair(conn, pair)
+
+
+PendingPairOrder = Literal[
+    "first.id ASC, second.id ASC",
+    "first.id ASC, second.id DESC",
+    "first.id DESC, second.id ASC",
+    "first.id DESC, second.id DESC"
+]
+PENDING_PAIR_ORDERS: list[PendingPairOrder] = [
+    "first.id DESC, second.id ASC",
+    "first.id ASC, second.id ASC",
+    "first.id ASC, second.id DESC",
+]
+
+
+def _select_pending_pairs(
+    conn: sqlite3.Connection,
+    order: PendingPairOrder = PENDING_PAIR_ORDERS[0],
+) -> Generator[PendingPair, None, None]:
+    result = conn.execute(
+        f"""
+        SELECT
+            first.id,
+            first.name,
+            first.emoji,
+            second.id,
+            second.name,
+            second.emoji
+        FROM element AS first
+        LEFT JOIN element AS second ON first.name <= second.name
+        LEFT JOIN pair ON pair.first_element_id = first.id AND pair.second_element_id = second.id
+        WHERE pair.id IS NULL
+        ORDER BY {order}
+        """,
+    )
+
+    for row in result:
+        first_id, first_name, first_emoji, second_id, second_name, second_emoji = row
+
+        yield PendingPair(
+            Element(first_name, first_emoji, first_id),
+            Element(second_name, second_emoji, second_id),
+        )
+
+
+def select_pending_pairs(order: PendingPairOrder) -> Generator[PendingPair, None, None]:
+    with connect() as conn:
+        yield from _select_pending_pairs(conn, order)
+
+
+def _element_count(conn: sqlite3.Connection) -> int:
+    (count,) = conn.execute("SELECT COUNT(*) FROM element").fetchone()
+    return count
+
+
+def _pair_count(conn: sqlite3.Connection) -> int:
+    (count,) = conn.execute("SELECT COUNT(*) FROM pair").fetchone()
+    return count
+
+
+def counts() -> tuple[int, int]:
+    with connect() as conn:
+        return _element_count(conn), _pair_count(conn)
+
+
+def _select_elements_and_discovered(
+    conn: sqlite3.Connection,
+) -> Generator[tuple[Element, bool], None, None]:
+    result = conn.execute(
+        """
+        SELECT
+            e.name,
+            e.emoji,
+            e.id,
+            MAX(p.result_element_id IS NOT NULL) AS is_discovery
+        FROM element e
+        LEFT JOIN pair p
+            ON p.result_element_id = e.id
+            AND p.is_discovery = TRUE
+        GROUP BY e.name, e.emoji, e.id
+        ORDER BY e.id ASC
+        """,
+    )
+
+    for row in result:
+        *e, is_discovery = row
+
+        yield Element(*e), is_discovery
+
+
+def select_elements_and_discovered() -> Generator[tuple[Element, bool], None, None]:
+    with connect() as conn:
+        return _select_elements_and_discovered(conn)
+
+
+with connect() as conn:
+    primary_elements = [
+        Element("Fire", "\N{FIRE}"),
+        Element("Earth", "\N{EARTH GLOBE EUROPE-AFRICA}"),
+        Element("Water", "\N{DROPLET}"),
+        Element("Wind", "\N{WIND BLOWING FACE}\N{VARIATION SELECTOR-16}"),
+    ]
+
+    # The search order is "mostly deterministic" on the macroscopic scale
+    # so randomize the order of the primary elements so that everyone who runs
+    # this code gets one of 4! (factorial) possible "macroscopic routes"
+    random.shuffle(primary_elements)
+
+    for e in primary_elements:
+        _upsert_element(conn, e)

dump.py

@@ -0,0 +1,37 @@
+import json
+from textwrap import dedent
+
+import db.persistence as persistence
+
+
+def dump() -> None:
+    data = [
+        (element.emoji, element.name, discovered)
+        for element, discovered in persistence.select_elements_and_discovered()
+    ]
+
+    print(
+        dedent(
+            f"""
+            let data = {json.dumps(data)};
+            let storage = JSON.parse(localStorage.getItem("infinite-craft-data")) || {{}};
+            storage.elements = storage.elements || [];
+
+            const nameSet = new Set(storage.elements.map(element => element.text));
+
+            data.forEach(element => {{
+                let [emoji, name, discovered] = element;
+
+                if (!nameSet.has(name)) {{
+                    storage.elements.push({{ text: name, emoji: emoji, discovered: discovered }});
+                }}
+            }});
+
+            localStorage.setItem("infinite-craft-data", JSON.stringify(storage));
+            """,
+        ).strip(),
+    )
+
+
+if __name__ == "__main__":
+    dump()

misc.py

@@ -0,0 +1,73 @@
+import os
+import random
+from curl_cffi import requests
+
+def header_bank_and_impersonate(browser='chrome'):
+    langs = [
+        "vi,en-US;q=0.9,en;q=0.8",
+        "en-US,en;q=0.9",
+        "en-GB,en;q=0.8",
+        "fr-FR,fr;q=0.9,en;q=0.8",
+        "zh-CN,zh;q=0.9,en;q=0.8",
+        "ja,en;q=0.9",
+    ]
+    lang = random.choice(langs)
+    is_mobile = False  # Change to True for mobile headers
+    platform = "\"Android\"" if is_mobile else random.choice(["\"Windows\"", "\"macOS\"", "\"Linux\"", "\"Chrome OS\""])
+    secchua = (
+        "\"Not)A;Brand\";v=\"8\", \"Chromium\";v=\"138\", "
+        f"\"{'Google Chrome' if browser == 'chrome' else 'Microsoft Edge'}\";v=\"138\""
+    )
+    user_agent = (
+        "Mozilla/5.0 (Linux; Android 13; SM-G991B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Mobile Safari/537.36"
+        if is_mobile else
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36"
+    )
+
+    # Example query
+    a, b = "Aquarium", "Avalanche"
+
+    curl = rf"""
+curl 'https://neal.fun/api/infinite-craft/pair?first={a}&second={b}' \
+  -H 'accept: */*' \
+  -H 'accept-language: {lang}' \
+  -H 'referer: https://neal.fun/infinite-craft/' \
+  -H 'referrer-policy: strict-origin-when-cross-origin' \
+  -H 'sec-ch-ua: {secchua}' \
+  -H 'sec-ch-ua-mobile: {"?1" if is_mobile else "?0"}' \
+  -H 'sec-ch-ua-platform: {platform}' \
+  -H 'sec-fetch-dest: empty' \
+  -H 'sec-fetch-mode: cors' \
+  -H 'sec-fetch-site: same-origin' \
+  -H 'user-agent: {user_agent}' \
+  --compressed
+"""
+    return curl.strip()
+
+def log_pair(pair):
+    with open('public/pair.txt', 'at+') as f:
+        f.write(pair + '\n')
+
+def trigger_push():
+    token = os.getenv('GITHUB_PAT')
+    repo = "ThongAccount/hf-dataset-pusher"
+    workflow = "update.yml"
+
+    url = f"https://api.github.com/repos/{repo}/actions/workflows/{workflow}/dispatches"
+    headers = {
+        "Accept": "application/vnd.github+json",
+        "Authorization": f"Bearer {token}",
+        "User-Agent": "hf-dataset-pusher",
+        "X-GitHub-Api-Version": "2022-11-28"
+    }
+    data = {"ref": "main", 
+            "inputs": {
+            "base_url": "https://thongcoder-infinite-craft-searcher.hf.space"
+           }
+           }
+
+    resp = requests.post(url, headers=headers, json=data)
+    if resp.ok:
+        print("✅ Triggered GitHub Action successfully.")
+    else:
+        print(f"❌ Failed to trigger Action: {resp.status_code} → {resp.text}")

models.py

@@ -0,0 +1,77 @@
+import re
+
+
+class Element:
+    def __init__(
+        self,
+        name: str,
+        emoji: str | None = None,
+        database_id: int | None = None,
+    ) -> None:
+        self.name = name
+        self.emoji = emoji or "\N{BLACK QUESTION MARK ORNAMENT}"
+        self.database_id = database_id
+
+    def __hash__(self) -> int:
+        return hash(self.name)
+
+    def __eq__(self, other: "Element") -> bool:
+        return self.name == other.name
+
+    def __str__(self) -> str:
+        return f"{self.emoji} {self.name}"
+
+    def __repr__(self) -> str:
+        return repr(str(self))
+
+    @property
+    def numeric(self) -> bool:
+        return re.search(r"\d", self.name) is not None
+
+
+class PendingPair:
+    def __init__(self, first: Element, second: Element) -> None:
+        self.first, self.second = (
+            (first, second) if first.name < second.name else (second, first)
+        )
+
+    def __hash__(self) -> int:
+        return hash((self.first, self.second))
+
+    def __eq__(self, other: "Pair") -> bool:
+        return self.first == other.first and self.second == other.second
+
+    def __str__(self) -> str:
+        return f"{self.first} + {self.second}"
+
+    def __repr__(self) -> str:
+        return f"{self.first!r} + {self.second!r}"
+
+    @property
+    def numeric(self) -> bool:
+        return self.first.numeric or self.second.numeric
+
+
+class Pair(PendingPair):
+    def __init__(
+        self,
+        first: Element,
+        second: Element,
+        result: Element,
+        is_discovery: bool | None = None,
+    ) -> None:
+        super().__init__(first, second)
+        self.result = result
+        self.is_discovery = is_discovery is True
+
+    def __str__(self) -> str:
+        addendum = " (New Discovery!)" if self.is_discovery else ""
+        return f"{super().__str__()} = {self.result}{addendum}"
+
+    def __repr__(self) -> str:
+        addendum = " (New Discovery!)" if self.is_discovery else ""
+        return f"{super().__repr__()} = {self.result!r}{addendum}"
+
+    @property
+    def elements(self) -> tuple[Element, Element, Element]:
+        return self.first, self.second, self.result

public/live-pairs.html

@@ -0,0 +1,30 @@
+<!-- public/live.html -->
+<!DOCTYPE html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <title>Live Pairing Stream</title>
+  <style>
+    body { font-family: sans-serif; padding: 1em; }
+    #log { white-space: pre-wrap; font-family: monospace; }
+  </style>
+</head>
+<body>
+  <h1>🔁 Infinite Craft: Live Pairing</h1>
+  <div id="log">Waiting for stream...</div>
+
+  <script>
+    const logDiv = document.getElementById("log");
+    const es = new EventSource("/stream-pair");
+
+    es.onmessage = (event) => {
+      logDiv.textContent = event.data + "\n" + logDiv.textContent;
+    };
+
+    es.onerror = () => {
+      logDiv.textContent = "❌ Connection lost. Refresh to retry.\n" + logDiv.textContent;
+      es.close();
+    };
+  </script>
+</body>
+</html>

requirements.txt

@@ -0,0 +1,3 @@
+curl_cffi
+fastapi
+uvicorn

run.py

@@ -0,0 +1,32 @@
+import concurrent.futures
+import os, curl_cffi.requests as requests
+import time
+from scan import scan
+
+
+def start_finder():
+    try:
+        url = 'https://huggingface.co/datasets/ThongCoder/infinite-craft/resolve/main/cache.sqlite'
+        filename = 'db/cache.sqlite'
+        root_dir = os.path.dirname(os.path.abspath(__file__))  # points to /app
+        dest_path = os.path.join(root_dir, filename)
+
+        print(f"Downloading from: {url}")
+        response = requests.get(url, stream=True)
+        response.raise_for_status()
+
+        with open(dest_path, "wb") as f:
+            for chunk in response.iter_content(chunk_size=8192):
+                f.write(chunk)
+
+        print(f"Saved to: {dest_path}")
+        time.sleep(.5)
+    except Exception as e: print(f'Error hit: {e}'); pass
+
+    while True:
+        try: scan(False, .2, 64)
+        except concurrent.futures._base.TimeoutError: continue
+        except Exception as e: print(f'Unexpected error:\n{e.with_traceback}'); break
+        
+if __name__ == "__main__":
+    start_finder()

scan.py

@@ -0,0 +1,202 @@
+import time
+from concurrent.futures import Future, ThreadPoolExecutor, as_completed
+from typing import Generator, TypeAlias
+
+import api
+import cloudflare
+from misc import log_pair, trigger_push
+import db.persistence as persistence
+from models import Pair, PendingPair
+
+Failed: TypeAlias = set[PendingPair]
+Futures: TypeAlias = dict[Future[Pair], PendingPair]
+Headers: TypeAlias = dict[str, str]
+
+
+def valid_pending_pairs(
+    allow_numbers: bool,
+    *,
+    failed: Failed,
+    futures: Futures,
+    order: persistence.PendingPairOrder,
+) -> Generator[PendingPair, None, None]:
+    for pending_pair in persistence.select_pending_pairs(order):
+        if not allow_numbers and pending_pair.numeric:
+            continue
+
+        if pending_pair in failed:
+            continue
+
+        if pending_pair in futures.values():
+            continue
+
+        yield pending_pair
+
+
+def queue_pair(
+    executor: ThreadPoolExecutor,
+    pending_pair: PendingPair,
+    futures: Futures,
+    *,
+    headers: Headers,
+) -> None:
+    futures[
+        executor.submit(
+            api.make_pair_exp_backoff,
+            pending_pair,
+            headers,
+            timeout=5,
+        )
+    ] = pending_pair
+
+
+def push_one_future(
+    executor: ThreadPoolExecutor,
+    futures: Futures,
+    *,
+    allow_numbers: bool,
+    failed: Failed,
+    headers: Headers,
+    order: persistence.PendingPairOrder,
+) -> bool:
+    for pending_pair in valid_pending_pairs(
+        allow_numbers,
+        failed=failed,
+        futures=futures,
+        order=order,
+    ):
+        queue_pair(executor, pending_pair, futures, headers=headers)
+        return True
+    return False
+
+
+def handle_completed_futures(
+    futures: Futures,
+    *,
+    failed: Failed,
+    timeout: float,
+) -> Generator[Pair | None, None, None]:
+    n_elements, n_pairs = persistence.counts()
+    log_line = f"Pairs: {n_pairs:,d}  Elements: {n_elements:,d}"
+    last_n_elements = n_elements
+    for future in as_completed(futures, timeout=timeout):
+        pending_pair = futures.pop(future)
+        try:
+            pair = future.result()
+        except TimeoutError:
+            print(f"[API TIMED OUT] {pending_pair}".ljust(len(log_line)))
+            print(log_line, end="\r")
+            failed.add(pending_pair)
+            yield None
+            continue
+        except Exception as e:
+            print(f"[API FAILED - {e!r}] {pending_pair}".ljust(len(log_line)))
+            print(log_line, end="\r")
+            failed.add(pending_pair)
+            yield None
+            continue
+
+        try:
+            persistence.record_pair(pair)
+        except Exception as e:
+            print(f"[DATABASE FAILED - {e!r}] {pair}".ljust(len(log_line)))
+            print(log_line, end="\r")
+            failed.add(pending_pair)
+            yield None
+            continue
+
+        yield pair
+
+        n_elements, n_pairs = persistence.counts()
+        log_line = f"Pairs: {n_pairs:,d}  Elements: {n_elements:,d}"
+
+        print(f"Pair #{n_pairs}: {str(pair)}")
+        log_pair(f"Pair #{n_pairs}: {str(pair)}")
+        if n_elements != last_n_elements:
+            res_name = pair.result.name
+            res_emoji = pair.result.emoji
+            res_id = pair.result.database_id
+            print(f'New element: {res_emoji} {res_name} (ID {res_id})')
+            log_pair(f'New element: {res_emoji} {res_name} (ID {res_id})')
+        if n_pairs % 10000 == 0:
+            print(f'Reached {n_pairs} pairs. Sending to DB.')
+            trigger_push()
+            time.sleep(90)
+            
+def now() -> float:
+    return time.perf_counter()
+
+
+def scan(allow_numbers: bool, seconds_per_request: float, threads: int) -> None:
+    threads = max(threads, 1)
+
+    headers: Headers = cloudflare.get_headers()
+    failed: Failed = set()
+    futures: Futures = {}
+
+    orders = persistence.PENDING_PAIR_ORDERS.copy()
+
+    with ThreadPoolExecutor(threads) as executor:
+
+        def shutdown() -> None:
+            executor.shutdown(False, cancel_futures=True)
+            incomplete_futures = [f for f in futures if not f.done()]
+            if not incomplete_futures:
+                return
+
+            n = len(incomplete_futures)
+
+            before = time.perf_counter()
+            print(f"[SHUTTING DOWN] 0/{n} threads terminated...", end="\r")
+            for i, _ in enumerate(as_completed(incomplete_futures), 1):
+                print(f"[SHUTTING DOWN] {i}/{n} threads terminated...", end="\r")
+            duration = 1000 * (time.perf_counter() - before)
+            print(f"[SHUTDOWN] {n} thread(s) completed in {duration:.2f} milliseconds.")
+
+        while True:
+            if len(futures) < threads * 2:
+                pushed = push_one_future(
+                    executor,
+                    futures,
+                    allow_numbers=allow_numbers,
+                    failed=failed,
+                    headers=headers,
+                    order=orders[0],
+                )
+
+                if not pushed:
+                    if failed:
+                        failed.clear()
+                        continue
+
+                    if not futures:
+                        print("Completed! All possible pairs have been made!")
+                        return
+
+            next_future_at = now() + seconds_per_request
+            try:
+                for pair in handle_completed_futures(
+                    futures,
+                    failed=failed,
+                    timeout=next_future_at - now(),
+                ):
+                    if not pair or pair.result.name.lower() == "nothing":
+                        orders.insert(0, orders.pop())
+            except TimeoutError:
+                pass
+            except Exception as e:
+                pass
+                
+            delay_remaining = next_future_at - now()
+            if delay_remaining < 0:
+                continue
+
+            try:
+                time.sleep(delay_remaining)
+            except:
+                shutdown()
+                raise
+
+
+if __name__ == "__main__":
+    scan(False, 0.25, 8)