wohnbot/handlers/howoge_handler.py

from .base_handler import BaseHandler
import logging
import asyncio

logger = logging.getLogger(__name__)

class HowogeHandler(BaseHandler):
    def __init__(self, browser_context):
        self.context = browser_context

    async def apply(self, listing: dict, result: dict) -> dict:
        import os
        from pathlib import Path
        DATA_DIR = Path("data/howoge")
        DATA_DIR.mkdir(parents=True, exist_ok=True)
        page = await self.context.new_page()
        try:
            logger.info(f"[HOWOGE] Opening page: {listing['link']}")
            response = await page.goto(listing["link"], wait_until="networkidle")
            logger.info("[HOWOGE] Page loaded")
            await asyncio.sleep(2)

            # Detect 404 by status or page title
            status = response.status if response else None
            page_title = await page.title()
            if status == 404 or (page_title and "404" in page_title):
                logger.warning(f"[HOWOGE] Listing is down (404): {listing['link']}")
                result["success"] = False
                result["message"] = "Listing is no longer available (404). Application impossible. Will not retry."
                result["deactivated"] = True
                await page.close()
                return result

            # Handle cookies
            try:
                cookie_btn = await page.query_selector('button:has-text("Akzeptieren"), button:has-text("Alle akzeptieren")')
                if cookie_btn and await cookie_btn.is_visible():
                    await cookie_btn.click()
                    logger.info("[HOWOGE] Dismissed cookie banner")
                    await asyncio.sleep(1)
            except: pass

            # Try to handle consent manager (consentmanager.net)
            try:
                consent_selectors = [
                    '#cmpbntyestxt', '.cmpboxbtnyes', 'a.cmpboxbtn.cmpboxbtnyes',
                    '#cmpwelcomebtnyes', '.cmptxt_btn_yes'
                ]
                for sel in consent_selectors:
                    consent_btn = await page.query_selector(sel)
                    if consent_btn and await consent_btn.is_visible():
                        await consent_btn.click()
                        logger.info("[HOWOGE] Dismissed consent manager")
                        await asyncio.sleep(1)
                        break
            except: pass

            # Look for "Besichtigung vereinbaren" button
            logger.info("[HOWOGE] Looking for 'Besichtigung vereinbaren' button...")
            selectors = [
                'a[href*="besichtigung-vereinbaren"]',
                'a:has-text("Besichtigung vereinbaren")',
                'button:has-text("Besichtigung vereinbaren")',
                'a:has-text("Anfragen")',
                'button:has-text("Anfragen")'
            ]
            apply_btn = None
            for sel in selectors:
                all_btns = await page.query_selector_all(sel)
                logger.info(f"[HOWOGE] Selector '{sel}' found {len(all_btns)} matches")
                for btn in all_btns:
                    try:
                        if await btn.is_visible():
                            apply_btn = btn
                            logger.info(f"[HOWOGE] Found visible button with selector '{sel}'")
                            break
                    except:
                        pass
                if apply_btn:
                    break

            if apply_btn:
                # Scroll the button into view and click
                logger.info("[HOWOGE] Found application button, scrolling into view...")
                await apply_btn.scroll_into_view_if_needed()
                await asyncio.sleep(0.5)
                logger.info("[HOWOGE] Clicking button...")
                await apply_btn.click()
                await asyncio.sleep(3)
                await page.wait_for_load_state("networkidle")
                logger.info("[HOWOGE] Clicked button, starting multi-step form process...")

                max_steps = 6  # safety limit
                for step in range(1, max_steps + 1):
                    logger.info(f"[HOWOGE] Processing step {step}")
                    await page.evaluate("window.scrollBy(0, 300)")
                    await asyncio.sleep(0.5)
                    email_field = await page.query_selector('input[name*="email" i]')
                    if email_field and await email_field.is_visible():
                        logger.info("[HOWOGE] Email field is visible - form is ready!")
                        break
                    checkboxes = await page.query_selector_all('input[type="checkbox"]')
                    clicked_checkbox = False
                    for checkbox in checkboxes:
                        try:
                            if await checkbox.is_visible() and not await checkbox.is_checked():
                                await checkbox.evaluate("el => el.click()")
                                clicked_checkbox = True
                                logger.info(f"[HOWOGE] Clicked checkbox in step {step}")
                                await asyncio.sleep(0.5)
                        except Exception as e:
                            logger.debug(f"[HOWOGE] Checkbox click failed: {e}")
                    if clicked_checkbox:
                        await asyncio.sleep(1)
                    screenshot_path = DATA_DIR / f"step{step}_{listing['id']}.png"
                    await page.screenshot(path=str(screenshot_path), full_page=True)
                    weiter_btns = await page.query_selector_all('button:has-text("Weiter")')
                    weiter_clicked = False
                    for btn in weiter_btns:
                        try:
                            if await btn.is_visible():
                                await btn.click()
                                weiter_clicked = True
                                logger.info(f"[HOWOGE] Clicked 'Weiter' button in step {step}")
                                await asyncio.sleep(2)
                                await page.wait_for_load_state("networkidle")
                                break
                        except Exception as e:
                            logger.debug(f"[HOWOGE] Weiter click failed: {e}")
                    if not weiter_clicked and not clicked_checkbox:
                        logger.warning(f"[HOWOGE] No action possible in step {step}, breaking")
                        break

                # Now try to fill the form
                logger.info("[HOWOGE] Attempting to fill form fields...")
                vorname_field = await page.query_selector('input[name*="firstName" i], input[name*="vorname" i]')
                nachname_field = await page.query_selector('input[name*="lastName" i], input[name*="nachname" i]')
                email_field = await page.query_selector('input[type="email"], input[name*="email" i]')
                form_filled = False
                if vorname_field and await vorname_field.is_visible():
                    await vorname_field.fill(os.environ.get("FORM_VORNAME", "Max"))
                    logger.info(f"[HOWOGE] Filled Vorname: {os.environ.get('FORM_VORNAME', 'Max')}")
                    form_filled = True
                else:
                    logger.warning("[HOWOGE] Vorname field not found or not visible")
                if nachname_field and await nachname_field.is_visible():
                    await nachname_field.fill(os.environ.get("FORM_NACHNAME", "Mustermann"))
                    logger.info(f"[HOWOGE] Filled Nachname: {os.environ.get('FORM_NACHNAME', 'Mustermann')}")
                    form_filled = True
                else:
                    logger.warning("[HOWOGE] Nachname field not found or not visible")
                if email_field and await email_field.is_visible():
                    await email_field.fill(os.environ.get("FORM_EMAIL", "test@example.com"))
                    logger.info(f"[HOWOGE] Filled Email: {os.environ.get('FORM_EMAIL', 'test@example.com')}")
                    form_filled = True
                else:
                    logger.warning("[HOWOGE] Email field not found or not visible")
                phone_field = await page.query_selector('input[type="tel"], input[name*="telefon" i], input[name*="phone" i]')
                if phone_field and await phone_field.is_visible():
                    await phone_field.fill(os.environ.get("FORM_PHONE", "0123456789"))
                    logger.info(f"[HOWOGE] Filled Phone: {os.environ.get('FORM_PHONE', '0123456789')}")
                screenshot_path2 = DATA_DIR / f"filled_{listing['id']}.png"
                await page.screenshot(path=str(screenshot_path2), full_page=True)
                logger.info(f"[HOWOGE] Saved filled form screenshot to {screenshot_path2}")
                if form_filled:
                    submit_btn = None
                    for selector in ['button:has-text("Anfrage senden")', 'button:has-text("Absenden")', 'button:has-text("Senden")']:
                        btn = await page.query_selector(selector)
                        if btn and await btn.is_visible():
                            submit_btn = btn
                            logger.info(f"[HOWOGE] Found submit button with selector: {selector}")
                            break
                    if submit_btn:
                        logger.info("[HOWOGE] Found submit button, clicking...")
                        await submit_btn.click()
                        await asyncio.sleep(3)
                        await page.wait_for_load_state("networkidle")
                        screenshot_path3 = DATA_DIR / f"submitted_{listing['id']}.png"
                        await page.screenshot(path=str(screenshot_path3))
                        logger.info(f"[HOWOGE] Saved post-submit screenshot to {screenshot_path3}")
                        content = await page.content()
                        if "erfolgreich" in content.lower() or "gesendet" in content.lower() or "danke" in content.lower() or "bestätigung" in content.lower():
                            result["success"] = True
                            result["message"] = "Application submitted successfully"
                            logger.info("[HOWOGE] Success! Confirmation message detected")
                        else:
                            result["success"] = False
                            result["message"] = "Form submitted but no confirmation detected"
                            logger.warning("[HOWOGE] Form submitted but no clear confirmation")
                    else:
                        result["success"] = False
                        result["message"] = "Form filled but no submit button found"
                        logger.warning("[HOWOGE] Could not find submit button")
                else:
                    result["success"] = False
                    result["message"] = "Could not find form fields to fill after navigating steps"
                    logger.warning("[HOWOGE] No form fields found after multi-step navigation")
            else:
                result["message"] = "No application button found"
                logger.warning("[HOWOGE] Could not find 'Besichtigung vereinbaren' button")
                screenshot_path = DATA_DIR / f"nobtn_{listing['id']}.png"
                await page.screenshot(path=str(screenshot_path))
                buttons = await page.query_selector_all('button, a.btn, a[class*="button"]')
                for btn in buttons[:10]:
                    try:
                        text = await btn.inner_text()
                        logger.info(f"[HOWOGE] Found button: {text[:50]}")
                    except:
                        pass
        except Exception as e:
            result["message"] = f"Error: {str(e)}"
            logger.error(f"[HOWOGE] Exception: {str(e)}")
            import traceback
            logger.error(traceback.format_exc())
            # Save debug HTML on error
            debug_html_path = DATA_DIR / f"debug_{listing['id']}.html"
            try:
                html = await page.content()
                with open(debug_html_path, "w", encoding="utf-8") as f:
                    f.write(html)
                logger.info(f"[HOWOGE] Saved debug HTML to {debug_html_path}")
            except Exception as html_e:
                logger.error(f"[HOWOGE] Failed to save debug HTML: {html_e}")
        finally:
            await page.close()
        return result