from handlers.base_handler import BaseHandler import logging import asyncio import os from pathlib import Path logger = logging.getLogger(__name__) class DegewoHandler(BaseHandler): def __init__(self, browser_context): self.context = browser_context async def apply(self, listing: dict, result: dict) -> dict: import os DATA_DIR = Path("data/degewo") DATA_DIR.mkdir(parents=True, exist_ok=True) page = await self.context.new_page() try: logger.info(f"[DEGEWO] Opening page: {listing['link']}") response = await page.goto(listing["link"], wait_until="networkidle") logger.info("[DEGEWO] Page loaded") await asyncio.sleep(2) # 404 detection status = response.status if response else None page_title = await page.title() if status == 404 or (page_title and "404" in page_title): logger.warning(f"[DEGEWO] Listing is down (404): {listing['link']}") result["success"] = False result["message"] = "Listing is no longer available (404). Application impossible. Will not retry." result["deactivated"] = True return result # Check for 'INSERAT DEAKTIVIERT' (deactivated listing) page_content = await page.content() if "INSERAT DEAKTIVIERT" in page_content or "Inserat deaktiviert" in page_content: logger.warning("[DEGEWO] Listing is deactivated (INSERAT DEAKTIVIERT detected), treating as 404") result["success"] = False result["message"] = "Listing deactivated (404)" result["deactivated"] = True # Mark for removal from retries await page.close() return result # Dismiss cookie banner try: cookie_btn = await page.query_selector('button:has-text("Alle akzeptieren"), #CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll') if cookie_btn and await cookie_btn.is_visible(): await cookie_btn.click() logger.info("[DEGEWO] Dismissed cookie banner") await asyncio.sleep(1) except Exception as e: logger.debug(f"[DEGEWO] Cookie banner dismiss failed: {e}") # Save debug HTML and screenshot before checking button try: html_content = await page.content() html_path = DATA_DIR / f"degewo_page_{listing['id']}.html" with open(html_path, 'w', encoding='utf-8') as f: f.write(html_content) logger.info(f"[DEGEWO] Saved page HTML to {html_path}") # Check if listing is no longer available if "nicht mehr verfügbar" in html_content or "Objekt nicht mehr verfügbar" in html_content: logger.warning(f"[DEGEWO] Listing no longer available") result["success"] = False result["message"] = "Listing no longer available" result["deactivated"] = True await page.close() return result except Exception as e: logger.warning(f"[DEGEWO] Could not save HTML: {e}") logger.info("[DEGEWO] Looking for kontaktieren button...") apply_btn = await page.query_selector('a:has-text("kontaktieren"), button:has-text("kontaktieren"), a:has-text("Kontaktieren"), button:has-text("Kontaktieren")') if apply_btn and await apply_btn.is_visible(): logger.info("[DEGEWO] Found kontaktieren button, clicking...") await apply_btn.click() await asyncio.sleep(3) # Degewo uses Wohnungshelden iframe for the application form iframe_element = await page.query_selector('iframe[src*="wohnungshelden.de"]') if iframe_element: iframe_url = await iframe_element.get_attribute('src') logger.info(f"[DEGEWO] Found Wohnungshelden iframe: {iframe_url}") iframe_page = await self.context.new_page() try: await iframe_page.goto(iframe_url, wait_until="networkidle") await asyncio.sleep(2) logger.info("[DEGEWO] Loaded Wohnungshelden application page") # Screenshot and HTML for debugging screenshot_path = DATA_DIR / f"degewo_wohnungshelden_{listing['id']}.png" await iframe_page.screenshot(path=str(screenshot_path), full_page=True) logger.info(f"[DEGEWO] Saved Wohnungshelden screenshot to {screenshot_path}") html_content = await iframe_page.content() html_path = DATA_DIR / f"degewo_wohnungshelden_{listing['id']}.html" with open(html_path, 'w', encoding='utf-8') as f: f.write(html_content) logger.info(f"[DEGEWO] Saved HTML to {html_path}") # Fill out Wohnungshelden form form_filled = False # Anrede (Salutation) try: salutation_dropdown = await iframe_page.query_selector('#salutation-dropdown, ng-select[id*="salutation"]') if salutation_dropdown: await salutation_dropdown.click() await asyncio.sleep(0.5) anrede_option = await iframe_page.query_selector(f'.ng-option:has-text("{os.environ.get("FORM_ANREDE", "Herr")}")') if anrede_option: await anrede_option.click() logger.info(f"[DEGEWO] Selected Anrede: {os.environ.get('FORM_ANREDE', 'Herr')}") form_filled = True except Exception as e: logger.warning(f"[DEGEWO] Could not set Anrede: {e}") # Vorname try: vorname_field = await iframe_page.query_selector('#firstName') if vorname_field: await vorname_field.fill(os.environ.get("FORM_VORNAME", "Max")) logger.info(f"[DEGEWO] Filled Vorname: {os.environ.get('FORM_VORNAME', 'Max')}") form_filled = True except Exception as e: logger.warning(f"[DEGEWO] Could not fill Vorname: {e}") # Nachname try: nachname_field = await iframe_page.query_selector('#lastName') if nachname_field: await nachname_field.fill(os.environ.get("FORM_NACHNAME", "Mustermann")) logger.info(f"[DEGEWO] Filled Nachname: {os.environ.get('FORM_NACHNAME', 'Mustermann')}") form_filled = True except Exception as e: logger.warning(f"[DEGEWO] Could not fill Nachname: {e}") # E-Mail try: email_field = await iframe_page.query_selector('#email') if email_field: await email_field.fill(os.environ.get("FORM_EMAIL", "test@example.com")) logger.info(f"[DEGEWO] Filled E-Mail: {os.environ.get('FORM_EMAIL', 'test@example.com')}") form_filled = True except Exception as e: logger.warning(f"[DEGEWO] Could not fill E-Mail: {e}") # Telefonnummer try: tel_field = await iframe_page.query_selector('input[id*="telefonnummer"]') if tel_field: await tel_field.fill(os.environ.get("FORM_PHONE", "0123456789")) logger.info(f"[DEGEWO] Filled Telefon: {os.environ.get('FORM_PHONE', '0123456789')}") form_filled = True except Exception as e: logger.warning(f"[DEGEWO] Could not fill Telefon: {e}") # Anzahl einziehende Personen try: personen_field = await iframe_page.query_selector('input[id*="numberPersonsTotal"]') if personen_field: await personen_field.fill(os.environ.get("FORM_PERSONS", "1")) logger.info(f"[DEGEWO] Filled Anzahl Personen: {os.environ.get('FORM_PERSONS', '1')}") form_filled = True except Exception as e: logger.warning(f"[DEGEWO] Could not fill Anzahl Personen: {e}") # "Für sich selbst" dropdown try: selbst_dropdown = await iframe_page.query_selector('ng-select[id*="fuer_wen"]') if selbst_dropdown: await selbst_dropdown.click() await asyncio.sleep(0.5) selbst_option = await iframe_page.query_selector('.ng-option:has-text("Für mich selbst"), .ng-option:has-text("selbst")') if selbst_option: await selbst_option.click() logger.info("[DEGEWO] Selected: Für mich selbst") form_filled = True except Exception as e: logger.warning(f"[DEGEWO] Could not set 'Für sich selbst': {e}") await asyncio.sleep(1) # Take screenshot after filling form screenshot_path = DATA_DIR / f"degewo_form_filled_{listing['id']}.png" await iframe_page.screenshot(path=str(screenshot_path), full_page=True) logger.info(f"[DEGEWO] Saved filled form screenshot to {screenshot_path}") # Try to submit try: submit_selectors = [ 'button[type="submit"]', 'input[type="submit"]', 'button:has-text("Absenden")', 'button:has-text("Senden")', 'button:has-text("Anfrage")', 'button:has-text("Bewerben")', 'button:has-text("Submit")', '.btn-primary', '.submit-btn', ] submit_btn = None for selector in submit_selectors: submit_btn = await iframe_page.query_selector(selector) if submit_btn and await submit_btn.is_visible(): logger.info(f"[DEGEWO] Found submit button with selector: {selector}") break submit_btn = None if submit_btn: await submit_btn.click() logger.info("[DEGEWO] Clicked submit button") await asyncio.sleep(3) # Take screenshot after submission screenshot_path = DATA_DIR / f"degewo_submitted_{listing['id']}.png" await iframe_page.screenshot(path=str(screenshot_path), full_page=True) logger.info(f"[DEGEWO] Saved submission screenshot to {screenshot_path}") result["success"] = True result["message"] = "Application submitted via Wohnungshelden" else: result["success"] = False result["message"] = "Wohnungshelden form loaded but submit button not found" logger.warning("[DEGEWO] Submit button not found in Wohnungshelden form") except Exception as e: result["success"] = False result["message"] = f"Wohnungshelden submit error: {str(e)}" logger.warning(f"[DEGEWO] Submit error: {e}") finally: await iframe_page.close() else: logger.warning("[DEGEWO] Wohnungshelden iframe not found, trying direct form...") screenshot_path = DATA_DIR / f"degewo_noiframe_{listing['id']}.png" await page.screenshot(path=str(screenshot_path), full_page=True) html_content = await page.content() html_path = DATA_DIR / "degewo_debug.html" with open(html_path, 'w', encoding='utf-8') as f: f.write(html_content) result["success"] = False result["message"] = "Wohnungshelden iframe not found on page" else: result["message"] = "No kontaktieren button found" logger.warning("[DEGEWO] Could not find kontaktieren button") screenshot_path = DATA_DIR / f"degewo_nobtn_{listing['id']}.png" await page.screenshot(path=str(screenshot_path), full_page=True) await page.close() return result except Exception as e: result["success"] = False result["message"] = f"Error: {str(e)}" logger.error(f"[DEGEWO] Exception: {str(e)}") await page.close() return result