import requests, json, time

from datetime import datetime

from bs4 import BeautifulSoup

import difflib



class WaybackScraper:

    CDX_API = "http://web.archive.org/cdx/search/cdx"

    WEB_BASE = "http://web.archive.org/web"



    def __init__(self):

        self.session = requests.Session()

        self.session.headers.update({'User-Agent': 'WaybackResearch/1.0'})



    def get_snapshots(self, url, from_date=None, to_date=None, limit=1000):

        params = {'url': url, 'output': 'json', 'limit': limit,

                  'fl': 'timestamp,original,statuscode,mimetype,length'}

        if from_date: params['from'] = from_date

        if to_date: params['to'] = to_date

        resp = self.session.get(self.CDX_API, params=params)

        data = resp.json()

        if len(data) <= 1: return []

        headers = data[0]

        return [dict(zip(headers, row)) for row in data[1:]]



    def get_page(self, url, timestamp):

        return self.session.get(f"{self.WEB_BASE}/{timestamp}id_/{url}").text

class EvolutionTracker:

    def __init__(self, scraper):

        self.scraper = scraper



    def track_changes(self, url, selector, samples=10):

        snaps = self.scraper.get_snapshots(url)

        if not snaps: return []

        step = max(1, len(snaps) // samples)

        versions = []

        for snap in snaps[::step]:

            try:

                html = self.scraper.get_page(url, snap['timestamp'])

                soup = BeautifulSoup(html, 'html.parser')

                el = soup.select_one(selector)

                if el:

                    versions.append({

                        'timestamp': snap['timestamp'],

                        'date': datetime.strptime(snap['timestamp'][:8], '%Y%m%d').isoformat(),

                        'content': el.get_text(strip=True)[:500]

                    })

                time.sleep(1)

            except Exception as e:

                print(f"Error: {e}")

        return versions



    def compare(self, versions):

        changes = []

        for i in range(1, len(versions)):

            old = versions[i-1]['content'].splitlines()

            new = versions[i]['content'].splitlines()

            diff = list(difflib.unified_diff(old, new, lineterm=''))

            if diff:

                changes.append({'from': versions[i-1]['date'], 'to': versions[i]['date'],

                                'diff_lines': len(diff)})

        return changes

class BulkCollector:

    def __init__(self, scraper):

        self.scraper = scraper



    def domain_history(self, domain):

        params = {'url': domain, 'output': 'json', 'matchType': 'domain',

                  'fl': 'timestamp,original,statuscode', 'collapse': 'urlkey',

                  'filter': 'statuscode:200'}

        data = self.scraper.session.get(self.scraper.CDX_API, params=params).json()

        if len(data) <= 1: return []

        return [dict(zip(data[0], row)) for row in data[1:]]



    def download_snapshot(self, domain, timestamp, output_dir='wayback_data'):

        import os

        os.makedirs(output_dir, exist_ok=True)

        urls = self.domain_history(domain)

        count = 0

        for entry in urls[:100]:

            try:

                content = self.scraper.get_page(entry['original'], timestamp)

                path = entry['original'].replace(f'https://{domain}','').replace(f'http://{domain}','')

                path = path.strip('/') or 'index.html'

                fp = os.path.join(output_dir, path)

                os.makedirs(os.path.dirname(fp), exist_ok=True)

                with open(fp, 'w') as f: f.write(content)

                count += 1

                time.sleep(1)

            except: pass

        return count

def track_pricing(scraper, url, selector):

    tracker = EvolutionTracker(scraper)

    versions = tracker.track_changes(url, selector, samples=20)

    print(f"Found {len(versions)} snapshots")

    for v in versions:

        print(f"  {v['date']}: {v['content'][:100]}")

    changes = tracker.compare(versions)

    print(f"Detected {len(changes)} changes")

    return versions, changes



wb = WaybackScraper()

track_pricing(wb, 'https://example.com/pricing', '.pricing-table')