from .base_handler import BaseHandler import logging import asyncio import os from pathlib import Path logger = logging.getLogger(__name__) DATA_DIR = Path("data/wbm") DATA_DIR.mkdir(parents=True, exist_ok=True) class WBMHandler(BaseHandler): def __init__(self, browser_context): self.context = browser_context async def apply(self, listing: dict, result: dict) -> dict: page = await self.context.new_page() try: logger.info(f"[WBM] Opening listing overview page: {listing['link']}") await page.goto(listing["link"], wait_until="networkidle") logger.info("[WBM] Overview page loaded") await asyncio.sleep(2) # Always handle cookies and consent before anything else await self.handle_cookies(page) await self.handle_consent(page) # Save HTML after modal handling for debugging try: html_content = await page.content() with open(DATA_DIR / "wbm_debug.html", "w", encoding="utf-8") as f: f.write(html_content) except Exception as e: logger.warning(f"[WBM] Could not save debug HTML: {e}") # 404/permanent fail detection error_texts = [ "Keine passenden Angebote gefunden", "Das Angebot existiert nicht mehr", "Die gewünschte Seite konnte nicht gefunden werden", "404", "Es wurden keine Immobilien gefunden" ] page_text = await page.text_content('body') if page_text: for err in error_texts: if err in page_text: result["deactivated"] = True result["message"] = "Listing is no longer available (404 detected on WBM)." logger.warning(f"[WBM] Permanent fail: {err}") await page.close() return result # Check if we landed on a generic overview/search page with multiple listings # This happens when the listing link is wrong or the listing was removed current_url = page.url # The overview page is at /angebote/ without /details/, and shows text like "X Mietwohnungen in Berlin" if '/angebote/' in current_url and '/details/' not in current_url: # Check for the heading pattern "X Mietwohnungen in Berlin" which appears on overview pages overview_heading = await page.query_selector('h2:has-text("Mietwohnungen in Berlin"), h3:has-text("Mietwohnungen in Berlin")') if overview_heading: result["deactivated"] = True result["message"] = "Redirected to generic overview page - listing no longer exists" logger.warning(f"[WBM] Landed on overview page (/angebote/) instead of specific listing detail") await page.screenshot(path=DATA_DIR / f"wbm_overview_redirect_{listing['id']}.png") await page.close() return result # Check if we're already on the detail page (URL contains '/details/') if '/details/' not in current_url: # Find and follow the 'Details' link to the detail page logger.info("[WBM] Looking for 'Details' link to open detail page...") detail_link = None detail_selectors = [ 'a.btn.sign[title="Details"]', 'a.immo-button-cta[title="Details"]', 'a[title="Details"]', ] for sel in detail_selectors: links = await page.query_selector_all(sel) logger.info(f"[WBM] Selector '{sel}' found {len(links)} matches for details link") for link in links: try: if await link.is_visible(): detail_link = link break except Exception as e: logger.warning(f"[WBM] Error checking details link visibility: {e}") if detail_link: break if not detail_link: result["message"] = "No details link found on overview page." await page.close() return result # Click the details link and wait for navigation logger.info("[WBM] Clicking details link to open detail page...") await detail_link.click() await page.wait_for_load_state("networkidle") await asyncio.sleep(2) else: logger.info("[WBM] Already on detail page, skipping details link search") # Save HTML of detail page for debugging try: html_content = await page.content() with open(DATA_DIR / "wbm_detail_debug.html", "w", encoding="utf-8") as f: f.write(html_content) except Exception as e: logger.warning(f"[WBM] Could not save detail debug HTML: {e}") # Look for application button on detail page logger.info("[WBM] Looking for application button on detail page...") selectors = [ 'button.btn.btn-primary[type="submit"]', # The actual form submit button 'button:has-text("Anfrage absenden")', 'a[href*="expose-anfordern"]', 'a[href*="bewerben"]', 'a:has-text("Anfragen")', 'button:has-text("Interesse")', 'a:has-text("Bewerben")', 'button:has-text("Bewerben")', 'button.btn', ] apply_btn = None for sel in selectors: all_btns = await page.query_selector_all(sel) logger.info(f"[WBM] Selector '{sel}' found {len(all_btns)} matches on detail page") for btn in all_btns: try: if await btn.is_visible(): apply_btn = btn logger.info(f"[WBM] Found visible application button with selector '{sel}' on detail page") break except Exception as e: logger.warning(f"[WBM] Error checking button visibility: {e}") if apply_btn: break if apply_btn: logger.info("[WBM] Found submit button, scrolling form into view...") await apply_btn.scroll_into_view_if_needed() await asyncio.sleep(1) # Handle cookies/consent again after scrolling (modals may reappear) await self.handle_cookies(page) await self.handle_consent(page) await asyncio.sleep(1) # Fill out the form fields before clicking submit logger.info("[WBM] Filling out application form...") form_filled = False try: # Anrede dropdown anrede_select = await page.query_selector('select[name*="anrede" i]') if anrede_select: await anrede_select.select_option(os.getenv("FORM_ANREDE", "Frau")) logger.debug("[WBM] Filled Anrede") form_filled = True # Name (Nachname) name_input = await page.query_selector('input[name*="name" i]:not([name*="vorname" i])') if name_input: await name_input.fill(os.getenv("FORM_NACHNAME", "")) logger.debug("[WBM] Filled Name") form_filled = True # Vorname vorname_input = await page.query_selector('input[name*="vorname" i]') if vorname_input: await vorname_input.fill(os.getenv("FORM_VORNAME", "")) logger.debug("[WBM] Filled Vorname") form_filled = True # Email (use ID or specific field name pattern) email_input = await page.query_selector('input#powermail_field_e_mail, input[name*="[e_mail]"], input[name*="[email]"]') if email_input: await email_input.fill(os.getenv("FORM_EMAIL", "")) logger.debug("[WBM] Filled Email") form_filled = True # Telefon phone_input = await page.query_selector('input[name*="telefon" i]') if phone_input: await phone_input.fill(os.getenv("FORM_PHONE", "")) logger.debug("[WBM] Filled Phone") # Strasse strasse_input = await page.query_selector('input[name*="strasse" i]') if strasse_input: await strasse_input.fill(os.getenv("FORM_STRASSE", "")) logger.debug("[WBM] Filled Strasse") # Hausnummer hausnummer_input = await page.query_selector('input[name*="hausnummer" i]') if hausnummer_input: await hausnummer_input.fill(os.getenv("FORM_HAUSNUMMER", "")) logger.debug("[WBM] Filled Hausnummer") # PLZ plz_input = await page.query_selector('input[name*="plz" i]') if plz_input: await plz_input.fill(os.getenv("FORM_PLZ", "")) logger.debug("[WBM] Filled PLZ") # Ort ort_input = await page.query_selector('input[name*="ort" i]') if ort_input: await ort_input.fill(os.getenv("FORM_ORT", "")) logger.debug("[WBM] Filled Ort") # Datenschutz checkbox - use force click or click the label to avoid interception datenschutz_checkbox = await page.query_selector('input[name*="datenschutz" i][type="checkbox"]') if datenschutz_checkbox: is_checked = await datenschutz_checkbox.is_checked() if not is_checked: # Try clicking the label first, fall back to force click on input datenschutz_label = await page.query_selector('label[for]:has(input[name*="datenschutz" i])') if datenschutz_label: await datenschutz_label.click() logger.debug("[WBM] Clicked Datenschutz label") else: await datenschutz_checkbox.click(force=True) logger.debug("[WBM] Force-clicked Datenschutz checkbox") if not form_filled: logger.error("[WBM] No form fields found - form may not be visible") result["message"] = "Form not found or not visible" await page.screenshot(path=DATA_DIR / f"wbm_no_form_{listing['id']}.png") await page.close() return result logger.info("[WBM] Form filled successfully") await asyncio.sleep(1) # Save screenshot before submit await page.screenshot(path=DATA_DIR / f"wbm_before_submit_{listing['id']}.png") except Exception as e: logger.error(f"[WBM] Error filling form: {e}") result["message"] = f"Error filling form: {e}" await page.screenshot(path=DATA_DIR / f"wbm_form_error_{listing['id']}.png") await page.close() return result logger.info("[WBM] Clicking submit button...") await apply_btn.click() await asyncio.sleep(3) # --- Post-click confirmation logic --- logger.info("[WBM] Clicked submit button, checking for confirmation...") # Save screenshot and HTML after click try: await page.screenshot(path=DATA_DIR / "wbm_after_apply.png") logger.info("[WBM] Saved screenshot after application click.") except Exception as e: logger.warning(f"[WBM] Could not save screenshot: {e}") try: html_after = await page.content() with open(DATA_DIR / "wbm_after_apply.html", "w", encoding="utf-8") as f: f.write(html_after) logger.info("[WBM] Saved HTML after application click.") except Exception as e: logger.warning(f"[WBM] Could not save HTML after apply: {e}") # Look for confirmation message on the page confirmation_selectors = [ 'text="Vielen Dank"', 'text="Ihre Anfrage wurde gesendet"', 'text="Bestätigung"', 'div:has-text("Vielen Dank")', 'div:has-text("Ihre Anfrage wurde gesendet")', ] confirmed = False for sel in confirmation_selectors: try: el = await page.query_selector(sel) if el and await el.is_visible(): logger.info(f"[WBM] Found confirmation element: {sel}") confirmed = True break except Exception as e: logger.debug(f"[WBM] Error checking confirmation selector {sel}: {e}") if confirmed: result["success"] = True result["message"] = "Application submitted and confirmation detected." else: logger.warning("[WBM] No confirmation message detected after application click.") result["success"] = False result["message"] = "Clicked application button, but no confirmation detected. Check screenshot and HTML." else: result["message"] = "No application button found on detail page." except Exception as e: result["message"] = f"Error during application: {e}" logger.error(f"[WBM] Application error: {e}") finally: await page.close() return result