SEO Performance Analysis Tool: AI-Powered SEO Insights with Complex Web Scraping

# lighthouse.py
def get_lighthouse(target_url: str):
    sbr_connection = ChromiumRemoteConnection(SBR_WEBDRIVER, 'goog', 'chrome')
    driver = Remote(sbr_connection, options=ChromeOptions())

    try:
        # Navigate to PageSpeed Insights
        encoded_url = f"https://pagespeed.web.dev/analysis?url={target_url}"
        driver.get(encoded_url)

        # Challenge 1: Wait for dynamic content loading
        WebDriverWait(driver, 60).until(
            EC.presence_of_element_located((By.CLASS_NAME, "lh-report"))
        )

        # Challenge 2: Handle tab switching for desktop analysis
        desktop_tab = WebDriverWait(driver, 20).until(
            EC.element_to_be_clickable((By.ID, "desktop_tab"))
        )
        actions = ActionChains(driver)
        actions.move_to_element(desktop_tab).click().perform()

        # Challenge 3: Verify report content changed
        WebDriverWait(driver, 20).until(
            lambda driver: driver.find_element(By.CLASS_NAME, "lh-report").text != report_text
        )

# compare_pages.py - Competitor Content Access
def fetch_html_content(url: str) -> tuple:
    try:
        # Ensure the URL has a proper scheme
        if not url.startswith(('http://', 'https://')):
            url = 'https://' + url

        # Brightdata API configuration
        api_url = "https://api.brightdata.com/request"
        headers = {
            "Content-Type": "application/json",
            "Authorization": f"Bearer {get_api_key('BRIGHTDATA_API_KEY')}"
        }
        payload = {
            "zone": "web_unlocker1",
            "url": url,
            "format": "raw"
        }

        # Make request to Brightdata API
        response = requests.post(api_url, json=payload, headers=headers)

        if response.status_code == 200:
            html_content = response.text
            soup = BeautifulSoup(html_content, 'html.parser')
            tags = soup.find_all(['h1', 'h2', 'h3', 'p'])
            collected_html = ''.join(str(tag) for tag in tags)
            return url, collected_html
    except Exception as e:
        print(f"Error fetching HTML content from {url}: {e}")
        return url, None

# compare_pages.py - Competitor Discovery
def get_top_competitor(keyword: str, our_domain: str) -> str:
    try:
        url = "https://api.brightdata.com/request"

        # Challenge: Get real-time SERP results and find relevant competitor
        encoded_keyword = requests.utils.quote(keyword)

        payload = {
            "zone": "serp_api1",
            "url": f"https://www.google.com/search?q={encoded_keyword}",
            "format": "raw"
        }

        headers = {
            "Authorization": f"Bearer {get_api_key('BRIGHTDATA_API_KEY')}",
            "Content-Type": "application/json"
        }

        response = requests.post(url, json=payload, headers=headers)

        if response.status_code == 200:
            # Parse search results with BeautifulSoup
            soup = BeautifulSoup(response.text, 'html.parser')
            all_data = soup.find_all("div", {"class": "g"})

            # Find first relevant competitor
            for result in all_data:
                link = result.find('a').get('href')
                if (link and 
                    link.find('https') != -1 and 
                    link.find('http') == 0 and 
                    our_domain not in link):
                    return link

    except Exception as e:
        st.error(f"Error finding competitor: {str(e)}")
        return None