from .base_handler import BaseHandler import logging import asyncio logger = logging.getLogger(__name__) class HowogeHandler(BaseHandler): def __init__(self, browser_context): self.context = browser_context async def apply(self, listing: dict, result: dict) -> dict: import os from pathlib import Path DATA_DIR = Path("data/howoge") DATA_DIR.mkdir(parents=True, exist_ok=True) page = await self.context.new_page() try: logger.info(f"[HOWOGE] Opening page: {listing['link']}") response = await page.goto(listing["link"], wait_until="networkidle") logger.info("[HOWOGE] Page loaded") await asyncio.sleep(2) # Detect 404 by status or page title status = response.status if response else None page_title = await page.title() if status == 404 or (page_title and "404" in page_title): logger.warning(f"[HOWOGE] Listing is down (404): {listing['link']}") result["success"] = False result["message"] = "Listing is no longer available (404). Application impossible. Will not retry." result["permanent_fail"] = True await page.close() return result # Handle cookies try: cookie_btn = await page.query_selector('button:has-text("Akzeptieren"), button:has-text("Alle akzeptieren")') if cookie_btn and await cookie_btn.is_visible(): await cookie_btn.click() logger.info("[HOWOGE] Dismissed cookie banner") await asyncio.sleep(1) except: pass # Try to handle consent manager (consentmanager.net) try: consent_selectors = [ '#cmpbntyestxt', '.cmpboxbtnyes', 'a.cmpboxbtn.cmpboxbtnyes', '#cmpwelcomebtnyes', '.cmptxt_btn_yes' ] for sel in consent_selectors: consent_btn = await page.query_selector(sel) if consent_btn and await consent_btn.is_visible(): await consent_btn.click() logger.info("[HOWOGE] Dismissed consent manager") await asyncio.sleep(1) break except: pass # Look for "Besichtigung vereinbaren" button logger.info("[HOWOGE] Looking for 'Besichtigung vereinbaren' button...") selectors = [ 'a[href*="besichtigung-vereinbaren"]', 'a:has-text("Besichtigung vereinbaren")', 'button:has-text("Besichtigung vereinbaren")', 'a:has-text("Anfragen")', 'button:has-text("Anfragen")' ] apply_btn = None for sel in selectors: all_btns = await page.query_selector_all(sel) logger.info(f"[HOWOGE] Selector '{sel}' found {len(all_btns)} matches") for btn in all_btns: try: if await btn.is_visible(): apply_btn = btn logger.info(f"[HOWOGE] Found visible button with selector '{sel}'") break except: pass if apply_btn: break if apply_btn: # Scroll the button into view and click logger.info("[HOWOGE] Found application button, scrolling into view...") await apply_btn.scroll_into_view_if_needed() await asyncio.sleep(0.5) logger.info("[HOWOGE] Clicking button...") await apply_btn.click() await asyncio.sleep(3) await page.wait_for_load_state("networkidle") logger.info("[HOWOGE] Clicked button, starting multi-step form process...") max_steps = 6 # safety limit for step in range(1, max_steps + 1): logger.info(f"[HOWOGE] Processing step {step}") await page.evaluate("window.scrollBy(0, 300)") await asyncio.sleep(0.5) email_field = await page.query_selector('input[name*="email" i]') if email_field and await email_field.is_visible(): logger.info("[HOWOGE] Email field is visible - form is ready!") break checkboxes = await page.query_selector_all('input[type="checkbox"]') clicked_checkbox = False for checkbox in checkboxes: try: if await checkbox.is_visible() and not await checkbox.is_checked(): await checkbox.evaluate("el => el.click()") clicked_checkbox = True logger.info(f"[HOWOGE] Clicked checkbox in step {step}") await asyncio.sleep(0.5) except Exception as e: logger.debug(f"[HOWOGE] Checkbox click failed: {e}") if clicked_checkbox: await asyncio.sleep(1) screenshot_path = DATA_DIR / f"step{step}_{listing['id']}.png" await page.screenshot(path=str(screenshot_path), full_page=True) weiter_btns = await page.query_selector_all('button:has-text("Weiter")') weiter_clicked = False for btn in weiter_btns: try: if await btn.is_visible(): await btn.click() weiter_clicked = True logger.info(f"[HOWOGE] Clicked 'Weiter' button in step {step}") await asyncio.sleep(2) await page.wait_for_load_state("networkidle") break except Exception as e: logger.debug(f"[HOWOGE] Weiter click failed: {e}") if not weiter_clicked and not clicked_checkbox: logger.warning(f"[HOWOGE] No action possible in step {step}, breaking") break # Now try to fill the form logger.info("[HOWOGE] Attempting to fill form fields...") vorname_field = await page.query_selector('input[name*="firstName" i], input[name*="vorname" i]') nachname_field = await page.query_selector('input[name*="lastName" i], input[name*="nachname" i]') email_field = await page.query_selector('input[type="email"], input[name*="email" i]') form_filled = False if vorname_field and await vorname_field.is_visible(): await vorname_field.fill(os.environ.get("FORM_VORNAME", "Max")) logger.info(f"[HOWOGE] Filled Vorname: {os.environ.get('FORM_VORNAME', 'Max')}") form_filled = True else: logger.warning("[HOWOGE] Vorname field not found or not visible") if nachname_field and await nachname_field.is_visible(): await nachname_field.fill(os.environ.get("FORM_NACHNAME", "Mustermann")) logger.info(f"[HOWOGE] Filled Nachname: {os.environ.get('FORM_NACHNAME', 'Mustermann')}") form_filled = True else: logger.warning("[HOWOGE] Nachname field not found or not visible") if email_field and await email_field.is_visible(): await email_field.fill(os.environ.get("FORM_EMAIL", "test@example.com")) logger.info(f"[HOWOGE] Filled Email: {os.environ.get('FORM_EMAIL', 'test@example.com')}") form_filled = True else: logger.warning("[HOWOGE] Email field not found or not visible") phone_field = await page.query_selector('input[type="tel"], input[name*="telefon" i], input[name*="phone" i]') if phone_field and await phone_field.is_visible(): await phone_field.fill(os.environ.get("FORM_PHONE", "0123456789")) logger.info(f"[HOWOGE] Filled Phone: {os.environ.get('FORM_PHONE', '0123456789')}") screenshot_path2 = DATA_DIR / f"filled_{listing['id']}.png" await page.screenshot(path=str(screenshot_path2), full_page=True) logger.info(f"[HOWOGE] Saved filled form screenshot to {screenshot_path2}") if form_filled: submit_btn = None for selector in ['button:has-text("Anfrage senden")', 'button:has-text("Absenden")', 'button:has-text("Senden")']: btn = await page.query_selector(selector) if btn and await btn.is_visible(): submit_btn = btn logger.info(f"[HOWOGE] Found submit button with selector: {selector}") break if submit_btn: logger.info("[HOWOGE] Found submit button, clicking...") await submit_btn.click() await asyncio.sleep(3) await page.wait_for_load_state("networkidle") screenshot_path3 = DATA_DIR / f"submitted_{listing['id']}.png" await page.screenshot(path=str(screenshot_path3)) logger.info(f"[HOWOGE] Saved post-submit screenshot to {screenshot_path3}") content = await page.content() if "erfolgreich" in content.lower() or "gesendet" in content.lower() or "danke" in content.lower() or "bestätigung" in content.lower(): result["success"] = True result["message"] = "Application submitted successfully" logger.info("[HOWOGE] Success! Confirmation message detected") else: result["success"] = False result["message"] = "Form submitted but no confirmation detected" logger.warning("[HOWOGE] Form submitted but no clear confirmation") else: result["success"] = False result["message"] = "Form filled but no submit button found" logger.warning("[HOWOGE] Could not find submit button") else: result["success"] = False result["message"] = "Could not find form fields to fill after navigating steps" logger.warning("[HOWOGE] No form fields found after multi-step navigation") else: result["message"] = "No application button found" logger.warning("[HOWOGE] Could not find 'Besichtigung vereinbaren' button") screenshot_path = DATA_DIR / f"nobtn_{listing['id']}.png" await page.screenshot(path=str(screenshot_path)) buttons = await page.query_selector_all('button, a.btn, a[class*="button"]') for btn in buttons[:10]: try: text = await btn.inner_text() logger.info(f"[HOWOGE] Found button: {text[:50]}") except: pass except Exception as e: result["message"] = f"Error: {str(e)}" logger.error(f"[HOWOGE] Exception: {str(e)}") import traceback logger.error(traceback.format_exc()) # Save debug HTML on error debug_html_path = DATA_DIR / f"debug_{listing['id']}.html" try: html = await page.content() with open(debug_html_path, "w", encoding="utf-8") as f: f.write(html) logger.info(f"[HOWOGE] Saved debug HTML to {debug_html_path}") except Exception as html_e: logger.error(f"[HOWOGE] Failed to save debug HTML: {html_e}") finally: await page.close() return result