wohnbot/handlers/stadtundland_handler.py

from .base_handler import BaseHandler

import logging
import asyncio
import os
from pathlib import Path

# Load environment variables for form fields and data dir
FORM_VORNAME = os.environ.get("FORM_VORNAME", "")
FORM_NACHNAME = os.environ.get("FORM_NACHNAME", "")
FORM_STRASSE = os.environ.get("FORM_STRASSE", "")
FORM_HAUSNUMMER = os.environ.get("FORM_HAUSNUMMER", "")
FORM_PLZ = os.environ.get("FORM_PLZ", "")
FORM_ORT = os.environ.get("FORM_ORT", "")
FORM_PHONE = os.environ.get("FORM_PHONE", "")
FORM_EMAIL = os.environ.get("FORM_EMAIL", "")
DATA_DIR = Path("data/stadtundland")
DATA_DIR.mkdir(parents=True, exist_ok=True)

logger = logging.getLogger(__name__)

class StadtUndLandHandler(BaseHandler):
    def __init__(self, browser_context):
        self.context = browser_context

    async def apply(self, listing: dict, result: dict) -> dict:
        page = await self.context.new_page()
        try:
            logger.info(f"[STADTUNDLAND] Opening page: {listing['link']}")
            await page.goto(listing["link"], wait_until="networkidle")
            logger.info("[STADTUNDLAND] Page loaded")
            await asyncio.sleep(2)

            # Always handle cookies and consent
            await self.handle_cookies(page)
            await self.handle_consent(page)
            await asyncio.sleep(1)

            # Check for 404 or error page early
            page_title = await page.title()
            page_content = await page.content()

            # Check for error messages
            if "schief gelaufen" in page_content.lower() or "schief gelaufen" in page_title.lower():
                logger.warning(f"[STADTUNDLAND] Error page detected (schief gelaufen) for {listing['link']}")
                result["success"] = False
                result["details"] = "Listing no longer available (error page)"
                await page.screenshot(path=str(DATA_DIR / f"stadtundland_404_{listing['id']}.png"))
                return result

            # Check for "nicht verfügbar" or similar messages
            if "nicht verfügbar" in page_content.lower() or "nicht mehr" in page_content.lower():
                logger.warning(f"[STADTUNDLAND] Listing not available: {listing['link']}")
                result["success"] = False
                result["details"] = "Listing no longer available"
                await page.screenshot(path=str(DATA_DIR / f"stadtundland_404_{listing['id']}.png"))
                return result

            # Scroll to form
            await page.evaluate("window.scrollBy(0, 500)")
            await asyncio.sleep(0.5)

            # Save HTML for debugging
            try:
                html_content = await page.content()
                with open(DATA_DIR / f"stadtundland_debug_{listing['id']}.html", "w", encoding="utf-8") as f:
                    f.write(html_content)
                logger.debug(f"[STADTUNDLAND] Saved debug HTML")
            except Exception as e:
                logger.warning(f"[STADTUNDLAND] Could not save debug HTML: {e}")

            # Fill out the embedded form directly
            form_filled = False
            try:
                # Vorname
                vorname_field = await page.query_selector('input[name="name"]')
                if vorname_field and await vorname_field.is_visible():
                    await vorname_field.fill(FORM_VORNAME)
                    logger.info(f"[STADTUNDLAND] Filled Vorname: {FORM_VORNAME}")
                    form_filled = True
                # Nachname
                nachname_field = await page.query_selector('input[name="surname"]')
                if nachname_field and await nachname_field.is_visible():
                    await nachname_field.fill(FORM_NACHNAME)
                    logger.info(f"[STADTUNDLAND] Filled Nachname: {FORM_NACHNAME}")
                    form_filled = True
                # Straße
                street_field = await page.query_selector('input[name="street"]')
                if street_field and await street_field.is_visible():
                    await street_field.fill(FORM_STRASSE)
                    logger.info(f"[STADTUNDLAND] Filled Straße: {FORM_STRASSE}")
                    form_filled = True
                # Hausnummer
                house_field = await page.query_selector('input[name="houseNo"]')
                if house_field and await house_field.is_visible():
                    await house_field.fill(FORM_HAUSNUMMER)
                    logger.info(f"[STADTUNDLAND] Filled Hausnummer: {FORM_HAUSNUMMER}")
                    form_filled = True
                # PLZ
                plz_field = await page.query_selector('input[name="postalCode"]')
                if plz_field and await plz_field.is_visible():
                    await plz_field.fill(FORM_PLZ)
                    logger.info(f"[STADTUNDLAND] Filled PLZ: {FORM_PLZ}")
                    form_filled = True
                # Ort
                city_field = await page.query_selector('input[name="city"]')
                if city_field and await city_field.is_visible():
                    await city_field.fill(FORM_ORT)
                    logger.info(f"[STADTUNDLAND] Filled Ort: {FORM_ORT}")
                    form_filled = True
                # Telefon
                phone_field = await page.query_selector('input[name="phone"]')
                if phone_field and await phone_field.is_visible():
                    await phone_field.fill(FORM_PHONE)
                    logger.info(f"[STADTUNDLAND] Filled Telefon: {FORM_PHONE}")
                    form_filled = True
                # E-Mail
                email_field = await page.query_selector('input[name="email"]')
                if email_field and await email_field.is_visible():
                    await email_field.fill(FORM_EMAIL)
                    logger.info(f"[STADTUNDLAND] Filled E-Mail: {FORM_EMAIL}")
                    form_filled = True
            except Exception as e:
                logger.warning(f"[STADTUNDLAND] Error filling form fields: {e}")

            # Click privacy checkbox
            try:
                privacy_checkbox = await page.query_selector('input[name="privacy"]')
                if privacy_checkbox and await privacy_checkbox.is_visible():
                    if not await privacy_checkbox.is_checked():
                        await privacy_checkbox.click()
                        logger.info("[STADTUNDLAND] Clicked privacy checkbox")
            except Exception as e:
                logger.warning(f"[STADTUNDLAND] Could not click privacy checkbox: {e}")

            # Click provision checkbox (optional)
            try:
                provision_checkbox = await page.query_selector('input[name="provision"]')
                if provision_checkbox and await provision_checkbox.is_visible():
                    if not await provision_checkbox.is_checked():
                        await provision_checkbox.click()
                        logger.info("[STADTUNDLAND] Clicked provision checkbox")
            except Exception as e:
                logger.warning(f"[STADTUNDLAND] Could not click provision checkbox: {e}")

            await asyncio.sleep(1)

            # Screenshot after filling form
            screenshot_path2 = DATA_DIR / f"stadtundland_filled_{listing['id']}.png"
            await page.screenshot(path=str(screenshot_path2), full_page=True)
            logger.info(f"[STADTUNDLAND] Saved filled form screenshot to {screenshot_path2}")

            # Submit form
            if form_filled:
                try:
                    pruefen_btn = await page.query_selector('button:has-text("Eingaben prüfen")')
                    if pruefen_btn and await pruefen_btn.is_visible():
                        await pruefen_btn.click()
                        logger.info("[STADTUNDLAND] Clicked 'Eingaben prüfen' button")
                        await asyncio.sleep(2)
                        await page.wait_for_load_state("networkidle")

                        # Screenshot after validation
                        screenshot_path3 = DATA_DIR / f"stadtundland_validated_{listing['id']}.png"
                        await page.screenshot(path=str(screenshot_path3), full_page=True)
                        logger.info(f"[STADTUNDLAND] Saved validation screenshot to {screenshot_path3}")

                        # Final submit
                        final_submit_selectors = [
                            'button:has-text("Absenden")',
                            'button:has-text("Senden")',
                            'button:has-text("Anfrage senden")',
                            'button:has-text("Bestätigen")',
                            'button[type="submit"]',
                        ]
                        final_btn = None
                        for selector in final_submit_selectors:
                            btn = await page.query_selector(selector)
                            if btn and await btn.is_visible():
                                final_btn = btn
                                logger.info(f"[STADTUNDLAND] Found final submit button: {selector}")
                                break
                        if final_btn:
                            await final_btn.click()
                            logger.info("[STADTUNDLAND] Clicked final submit button")
                            await asyncio.sleep(3)
                            await page.wait_for_load_state("networkidle")

                            # Screenshot after submission
                            screenshot_path4 = DATA_DIR / f"stadtundland_submitted_{listing['id']}.png"
                            await page.screenshot(path=str(screenshot_path4), full_page=True)
                            logger.info(f"[STADTUNDLAND] Saved submission screenshot to {screenshot_path4}")

                            # Check for confirmation
                            content = await page.content()
                            if any(word in content.lower() for word in ["erfolgreich", "gesendet", "danke", "bestätigung"]):
                                result["success"] = True
                                result["message"] = "Application submitted successfully"
                                logger.info("[STADTUNDLAND] Success! Confirmation message detected")
                            else:
                                result["success"] = True
                                result["message"] = "Form submitted"
                                logger.info("[STADTUNDLAND] Form submitted")
                        else:
                            result["success"] = False
                            result["message"] = "Validated but final submit button not found"
                            logger.warning("[STADTUNDLAND] Final submit button not found")
                    else:
                        result["success"] = False
                        result["message"] = "Form filled but 'Eingaben prüfen' button not found"
                        logger.warning("[STADTUNDLAND] 'Eingaben prüfen' button not found")
                except Exception as e:
                    result["success"] = False
                    result["message"] = f"Submit error: {str(e)}"
                    logger.warning(f"[STADTUNDLAND] Submit error: {e}")
            else:
                result["success"] = False
                result["message"] = "No form fields found on page"
                logger.warning("[STADTUNDLAND] Could not find form fields")

        except Exception as e:
            result["success"] = False
            result["message"] = f"Error: {str(e)}"
            logger.error(f"[STADTUNDLAND] Exception: {str(e)}")
        finally:
            await page.close()
        return result