wohnbot/handlers/degewo_handler.py

from handlers.base_handler import BaseHandler
import logging
import asyncio
import os
from pathlib import Path

logger = logging.getLogger(__name__)

class DegewoHandler(BaseHandler):
    def __init__(self, browser_context):
        self.context = browser_context

    async def apply(self, listing: dict, result: dict) -> dict:
        import os
        DATA_DIR = Path("data/degewo")
        DATA_DIR.mkdir(parents=True, exist_ok=True)
        page = await self.context.new_page()
        try:
            logger.info(f"[DEGEWO] Opening page: {listing['link']}")
            response = await page.goto(listing["link"], wait_until="networkidle")
            logger.info("[DEGEWO] Page loaded")
            await asyncio.sleep(2)

            # 404 detection
            status = response.status if response else None
            page_title = await page.title()
            if status == 404 or (page_title and "404" in page_title):
                logger.warning(f"[DEGEWO] Listing is down (404): {listing['link']}")
                result["success"] = False
                result["message"] = "Listing is no longer available (404). Application impossible. Will not retry."
                result["deactivated"] = True
                return result

            # Check for 'INSERAT DEAKTIVIERT' (deactivated listing)
            page_content = await page.content()
            if "INSERAT DEAKTIVIERT" in page_content or "Inserat deaktiviert" in page_content:
                logger.warning("[DEGEWO] Listing is deactivated (INSERAT DEAKTIVIERT detected), treating as 404")
                result["success"] = False
                result["message"] = "Listing deactivated (404)"
                result["deactivated"] = True  # Mark for removal from retries
                await page.close()
                return result

            # Dismiss cookie banner
            try:
                cookie_btn = await page.query_selector('button:has-text("Alle akzeptieren"), #CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll')
                if cookie_btn and await cookie_btn.is_visible():
                    await cookie_btn.click()
                    logger.info("[DEGEWO] Dismissed cookie banner")
                    await asyncio.sleep(1)
            except Exception as e:
                logger.debug(f"[DEGEWO] Cookie banner dismiss failed: {e}")

            logger.info("[DEGEWO] Looking for kontaktieren button...")
            apply_btn = await page.query_selector('a:has-text("kontaktieren"), button:has-text("kontaktieren"), a:has-text("Kontaktieren"), button:has-text("Kontaktieren")')
            if apply_btn and await apply_btn.is_visible():
                logger.info("[DEGEWO] Found kontaktieren button, clicking...")
                await apply_btn.click()
                await asyncio.sleep(3)

                # Degewo uses Wohnungshelden iframe for the application form
                iframe_element = await page.query_selector('iframe[src*="wohnungshelden.de"]')
                if iframe_element:
                    iframe_url = await iframe_element.get_attribute('src')
                    logger.info(f"[DEGEWO] Found Wohnungshelden iframe: {iframe_url}")

                    iframe_page = await self.context.new_page()
                    try:
                        await iframe_page.goto(iframe_url, wait_until="networkidle")
                        await asyncio.sleep(2)
                        logger.info("[DEGEWO] Loaded Wohnungshelden application page")

                        # Screenshot and HTML for debugging
                        screenshot_path = DATA_DIR / f"degewo_wohnungshelden_{listing['id']}.png"
                        await iframe_page.screenshot(path=str(screenshot_path), full_page=True)
                        logger.info(f"[DEGEWO] Saved Wohnungshelden screenshot to {screenshot_path}")
                        html_content = await iframe_page.content()
                        html_path = DATA_DIR / f"degewo_wohnungshelden_{listing['id']}.html"
                        with open(html_path, 'w', encoding='utf-8') as f:
                            f.write(html_content)
                        logger.info(f"[DEGEWO] Saved HTML to {html_path}")

                        # Fill out Wohnungshelden form
                        form_filled = False
                        # Anrede (Salutation)
                        try:
                            salutation_dropdown = await iframe_page.query_selector('#salutation-dropdown, ng-select[id*="salutation"]')
                            if salutation_dropdown:
                                await salutation_dropdown.click()
                                await asyncio.sleep(0.5)
                                anrede_option = await iframe_page.query_selector(f'.ng-option:has-text("{os.environ.get("FORM_ANREDE", "Herr")}")')
                                if anrede_option:
                                    await anrede_option.click()
                                    logger.info(f"[DEGEWO] Selected Anrede: {os.environ.get('FORM_ANREDE', 'Herr')}")
                                    form_filled = True
                        except Exception as e:
                            logger.warning(f"[DEGEWO] Could not set Anrede: {e}")

                        # Vorname
                        try:
                            vorname_field = await iframe_page.query_selector('#firstName')
                            if vorname_field:
                                await vorname_field.fill(os.environ.get("FORM_VORNAME", "Max"))
                                logger.info(f"[DEGEWO] Filled Vorname: {os.environ.get('FORM_VORNAME', 'Max')}")
                                form_filled = True
                        except Exception as e:
                            logger.warning(f"[DEGEWO] Could not fill Vorname: {e}")

                        # Nachname
                        try:
                            nachname_field = await iframe_page.query_selector('#lastName')
                            if nachname_field:
                                await nachname_field.fill(os.environ.get("FORM_NACHNAME", "Mustermann"))
                                logger.info(f"[DEGEWO] Filled Nachname: {os.environ.get('FORM_NACHNAME', 'Mustermann')}")
                                form_filled = True
                        except Exception as e:
                            logger.warning(f"[DEGEWO] Could not fill Nachname: {e}")

                        # E-Mail
                        try:
                            email_field = await iframe_page.query_selector('#email')
                            if email_field:
                                await email_field.fill(os.environ.get("FORM_EMAIL", "test@example.com"))
                                logger.info(f"[DEGEWO] Filled E-Mail: {os.environ.get('FORM_EMAIL', 'test@example.com')}")
                                form_filled = True
                        except Exception as e:
                            logger.warning(f"[DEGEWO] Could not fill E-Mail: {e}")

                        # Telefonnummer
                        try:
                            tel_field = await iframe_page.query_selector('input[id*="telefonnummer"]')
                            if tel_field:
                                await tel_field.fill(os.environ.get("FORM_PHONE", "0123456789"))
                                logger.info(f"[DEGEWO] Filled Telefon: {os.environ.get('FORM_PHONE', '0123456789')}")
                                form_filled = True
                        except Exception as e:
                            logger.warning(f"[DEGEWO] Could not fill Telefon: {e}")

                        # Anzahl einziehende Personen
                        try:
                            personen_field = await iframe_page.query_selector('input[id*="numberPersonsTotal"]')
                            if personen_field:
                                await personen_field.fill(os.environ.get("FORM_PERSONS", "1"))
                                logger.info(f"[DEGEWO] Filled Anzahl Personen: {os.environ.get('FORM_PERSONS', '1')}")
                                form_filled = True
                        except Exception as e:
                            logger.warning(f"[DEGEWO] Could not fill Anzahl Personen: {e}")

                        # "Für sich selbst" dropdown
                        try:
                            selbst_dropdown = await iframe_page.query_selector('ng-select[id*="fuer_wen"]')
                            if selbst_dropdown:
                                await selbst_dropdown.click()
                                await asyncio.sleep(0.5)
                                selbst_option = await iframe_page.query_selector('.ng-option:has-text("Für mich selbst"), .ng-option:has-text("selbst")')
                                if selbst_option:
                                    await selbst_option.click()
                                    logger.info("[DEGEWO] Selected: Für mich selbst")
                                    form_filled = True
                        except Exception as e:
                            logger.warning(f"[DEGEWO] Could not set 'Für sich selbst': {e}")

                        await asyncio.sleep(1)

                        # Take screenshot after filling form
                        screenshot_path = DATA_DIR / f"degewo_form_filled_{listing['id']}.png"
                        await iframe_page.screenshot(path=str(screenshot_path), full_page=True)
                        logger.info(f"[DEGEWO] Saved filled form screenshot to {screenshot_path}")

                        # Try to submit
                        try:
                            submit_selectors = [
                                'button[type="submit"]',
                                'input[type="submit"]',
                                'button:has-text("Absenden")',
                                'button:has-text("Senden")',
                                'button:has-text("Anfrage")',
                                'button:has-text("Bewerben")',
                                'button:has-text("Submit")',
                                '.btn-primary',
                                '.submit-btn',
                            ]

                            submit_btn = None
                            for selector in submit_selectors:
                                submit_btn = await iframe_page.query_selector(selector)
                                if submit_btn and await submit_btn.is_visible():
                                    logger.info(f"[DEGEWO] Found submit button with selector: {selector}")
                                    break
                                submit_btn = None

                            if submit_btn:
                                await submit_btn.click()
                                logger.info("[DEGEWO] Clicked submit button")
                                await asyncio.sleep(3)

                                # Take screenshot after submission
                                screenshot_path = DATA_DIR / f"degewo_submitted_{listing['id']}.png"
                                await iframe_page.screenshot(path=str(screenshot_path), full_page=True)
                                logger.info(f"[DEGEWO] Saved submission screenshot to {screenshot_path}")

                                result["success"] = True
                                result["message"] = "Application submitted via Wohnungshelden"
                            else:
                                result["success"] = False
                                result["message"] = "Wohnungshelden form loaded but submit button not found"
                                logger.warning("[DEGEWO] Submit button not found in Wohnungshelden form")
                        except Exception as e:
                            result["success"] = False
                            result["message"] = f"Wohnungshelden submit error: {str(e)}"
                            logger.warning(f"[DEGEWO] Submit error: {e}")
                    finally:
                        await iframe_page.close()
                else:
                    logger.warning("[DEGEWO] Wohnungshelden iframe not found, trying direct form...")
                    screenshot_path = DATA_DIR / f"degewo_noiframe_{listing['id']}.png"
                    await page.screenshot(path=str(screenshot_path), full_page=True)
                    html_content = await page.content()
                    html_path = DATA_DIR / "degewo_debug.html"
                    with open(html_path, 'w', encoding='utf-8') as f:
                        f.write(html_content)
                    result["success"] = False
                    result["message"] = "Wohnungshelden iframe not found on page"
            else:
                result["message"] = "No kontaktieren button found"
                logger.warning("[DEGEWO] Could not find kontaktieren button")
                screenshot_path = DATA_DIR / f"degewo_nobtn_{listing['id']}.png"
                await page.screenshot(path=str(screenshot_path), full_page=True)
            await page.close()
            return result
        except Exception as e:
            result["success"] = False
            result["message"] = f"Error: {str(e)}"
            logger.error(f"[DEGEWO] Exception: {str(e)}")
            await page.close()
            return result