| import requests
|
| from bs4 import BeautifulSoup
|
| import json
|
| import time
|
|
|
| SERPAPI_KEY = "c3e3e8fd8d12ca55d8a8954a14bf827f2d4261ef55373b381661f23b1440a2af"
|
|
|
| def google_search(query, num_results=1000):
|
| url = "https://serpapi.com/search"
|
| params = {
|
| "engine": "google",
|
| "q": query,
|
| "api_key": SERPAPI_KEY,
|
| "num": num_results
|
| }
|
| resp = requests.get(url, params=params)
|
| data = resp.json()
|
| links = []
|
| for result in data.get("organic_results", []):
|
| link = result.get("link")
|
| if link:
|
| links.append(link)
|
| return links
|
|
|
| def scrape_page(url):
|
| try:
|
| resp = requests.get(url, timeout=10, headers={"User-Agent": "Mozilla/5.0"})
|
| soup = BeautifulSoup(resp.content, "html.parser")
|
| text = soup.get_text(separator="\n", strip=True)
|
| return text[:20000]
|
| except Exception as e:
|
| return f"[SCRAPE ERROR] {e}"
|
|
|
| def scrape_topic_and_save(topic, filename="results.json"):
|
| links = google_search(topic)
|
| results = []
|
| for url in links:
|
| print(f"Scraping: {url}")
|
| content = scrape_page(url)
|
| results.append({"url": url, "content": content})
|
| time.sleep(0)
|
| with open(filename, "w", encoding="utf-8") as f:
|
| json.dump(results, f, ensure_ascii=False, indent=2)
|
| print(f"Saved {len(results)} results to {filename}")
|
|
|
| if __name__ == "__main__":
|
| topic = input("Enter topic to search: ")
|
| scrape_topic_and_save(topic) |
