from .base_handler import BaseHandler import logging import asyncio import os from pathlib import Path logger = logging.getLogger(__name__) DATA_DIR = Path("data/gewobag") DATA_DIR.mkdir(parents=True, exist_ok=True) class GewobagHandler(BaseHandler): def __init__(self, browser_context): self.context = browser_context async def apply(self, listing: dict, result: dict) -> dict: page = await self.context.new_page() try: logger.info(f"[GEWOBAG] Opening page: {listing['link']}") response = await page.goto(listing["link"], wait_until="networkidle") logger.info("[GEWOBAG] Page loaded") await asyncio.sleep(2) # Detect 404 by status, page title, or "nicht gefunden" message status = response.status if response else None page_title = await page.title() page_content = await page.content() is_404 = ( status == 404 or (page_title and "404" in page_title) or (page_title and "nicht gefunden" in page_title.lower()) or ("Mietangebot nicht gefunden" in page_content) ) if is_404: logger.warning(f"[GEWOBAG] Listing is down (404 or unavailable): {listing['link']}") result["success"] = False result["message"] = "Listing is no longer available (404). Application impossible. Will not retry." result["deactivated"] = True return result # Dismiss cookie banner try: cookie_btn = await page.query_selector('#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll, button:has-text("Alle akzeptieren")') if cookie_btn and await cookie_btn.is_visible(): await cookie_btn.click() logger.info("[GEWOBAG] Dismissed cookie banner") await asyncio.sleep(1) except: pass # Gewobag uses Wohnungshelden iframe directly on the page logger.info("[GEWOBAG] Looking for Wohnungshelden iframe...") iframe_element = await page.query_selector('iframe[src*="wohnungshelden.de"]') if iframe_element: iframe_url = await iframe_element.get_attribute('src') logger.info(f"[GEWOBAG] Found Wohnungshelden iframe: {iframe_url}") # Navigate to the iframe URL directly in a new page iframe_page = await self.context.new_page() try: await iframe_page.goto(iframe_url, wait_until="networkidle") await asyncio.sleep(2) logger.info("[GEWOBAG] Loaded Wohnungshelden application page") # Take screenshot screenshot_path = DATA_DIR / f"gewobag_wohnungshelden_{listing['id']}.png" await iframe_page.screenshot(path=str(screenshot_path), full_page=True) logger.info("[GEWOBAG] Saved Wohnungshelden screenshot") # Save HTML for debugging try: html_content = await iframe_page.content() with open(DATA_DIR / f"gewobag_wohnungshelden_{listing['id']}.html", "w", encoding="utf-8") as f: f.write(html_content) # Check for Wohnungshelden error messages if "Fehler beim Laden" in html_content or "Beim Laden der Gesuchsdaten ist ein Fehler aufgetreten" in html_content: logger.warning("[GEWOBAG] Wohnungshelden iframe shows error - may be transient, will retry") result["success"] = False result["message"] = "Wohnungshelden application form error - will retry later" await iframe_page.close() return result except Exception as e: logger.warning(f"[GEWOBAG] Could not save HTML: {e}") # Fill out Wohnungshelden form form_filled = False # Anrede (Salutation) - ng-select dropdown try: salutation_dropdown = await iframe_page.query_selector('#salutation-dropdown, ng-select[id*="salutation"]') if salutation_dropdown: await salutation_dropdown.click() await asyncio.sleep(0.5) anrede_option = await iframe_page.query_selector(f'.ng-option:has-text("{os.environ.get("FORM_ANREDE", "Herr")}")') if anrede_option: await anrede_option.click() logger.info(f"[GEWOBAG] Selected Anrede: {os.environ.get('FORM_ANREDE', 'Herr')}") form_filled = True except Exception as e: logger.warning(f"[GEWOBAG] Could not set Anrede: {e}") # Vorname (First name) try: vorname_field = await iframe_page.query_selector('#firstName') if vorname_field: await vorname_field.fill(os.environ.get("FORM_VORNAME", "")) logger.info(f"[GEWOBAG] Filled Vorname: {os.environ.get('FORM_VORNAME', '')}") form_filled = True except Exception as e: logger.warning(f"[GEWOBAG] Could not fill Vorname: {e}") # Nachname (Last name) try: nachname_field = await iframe_page.query_selector('#lastName') if nachname_field: await nachname_field.fill(os.environ.get("FORM_NACHNAME", "")) logger.info(f"[GEWOBAG] Filled Nachname: {os.environ.get('FORM_NACHNAME', '')}") form_filled = True except Exception as e: logger.warning(f"[GEWOBAG] Could not fill Nachname: {e}") # E-Mail try: email_field = await iframe_page.query_selector('#email') if email_field: await email_field.fill(os.environ.get("FORM_EMAIL", "")) logger.info(f"[GEWOBAG] Filled E-Mail: {os.environ.get('FORM_EMAIL', '')}") form_filled = True except Exception as e: logger.warning(f"[GEWOBAG] Could not fill E-Mail: {e}") # Telefonnummer try: tel_field = await iframe_page.query_selector('#phone-number, input[id*="telefonnummer"], input[id*="phone"]') if tel_field: await tel_field.fill(os.environ.get("FORM_PHONE", "")) logger.info(f"[GEWOBAG] Filled Telefon: {os.environ.get('FORM_PHONE', '')}") form_filled = True except Exception as e: logger.warning(f"[GEWOBAG] Could not fill Telefon: {e}") # Straße (Street) - formcontrolname="street" try: strasse_field = await iframe_page.query_selector('#street, input[formcontrolname="street"]') if strasse_field: await strasse_field.fill(os.environ.get("FORM_STRASSE", "")) logger.info(f"[GEWOBAG] Filled Straße: {os.environ.get('FORM_STRASSE', '')}") form_filled = True except Exception as e: logger.warning(f"[GEWOBAG] Could not fill Straße: {e}") # Hausnummer (House number) - formcontrolname="houseNumber" try: hausnr_field = await iframe_page.query_selector('input[formcontrolname="houseNumber"]') if hausnr_field: await hausnr_field.fill(os.environ.get("FORM_HAUSNUMMER", "")) logger.info(f"[GEWOBAG] Filled Hausnummer: {os.environ.get('FORM_HAUSNUMMER', '')}") form_filled = True except Exception as e: logger.warning(f"[GEWOBAG] Could not fill Hausnummer: {e}") # PLZ (Postal code) - formcontrolname="zipCode" try: plz_field = await iframe_page.query_selector('input[formcontrolname="zipCode"]') if plz_field: await plz_field.fill(os.environ.get("FORM_PLZ", "")) logger.info(f"[GEWOBAG] Filled PLZ: {os.environ.get('FORM_PLZ', '')}") form_filled = True except Exception as e: logger.warning(f"[GEWOBAG] Could not fill PLZ: {e}") # Ort (City) - formcontrolname="city" try: ort_field = await iframe_page.query_selector('input[formcontrolname="city"]') if ort_field: await ort_field.fill(os.environ.get("FORM_ORT", "")) logger.info(f"[GEWOBAG] Filled Ort: {os.environ.get('FORM_ORT', '')}") form_filled = True except Exception as e: logger.warning(f"[GEWOBAG] Could not fill Ort: {e}") # Gesamtzahl der einziehenden Personen (Erwachsene + Kinder) - REQUIRED try: anzahl_erwachsene = int(os.environ.get("FORM_ADULTS", "1")) anzahl_kinder = int(os.environ.get("FORM_CHILDREN", "0")) total_persons = str(anzahl_erwachsene + anzahl_kinder) total_input = await iframe_page.query_selector('#formly_2_input_gewobag_gesamtzahl_der_einziehenden_personen_erwachsene_und_kinder_0') if total_input: await total_input.fill(total_persons) logger.info(f"[GEWOBAG] Filled Gesamtzahl Personen: {total_persons}") form_filled = True except Exception as e: logger.warning(f"[GEWOBAG] Could not fill Gesamtzahl Personen: {e}") # "Für wen wird die Wohnungsanfrage gestellt?" dropdown - REQUIRED # Select "für mich selbst" (for myself) try: dropdown_input = await iframe_page.query_selector('#formly_9_select_gewobag_fuer_wen_wird_die_wohnungsanfrage_gestellt_0') if dropdown_input: await dropdown_input.click() await iframe_page.wait_for_timeout(300) # Select the first option "für mich selbst" for_myself_option = await iframe_page.query_selector('.ng-option:has-text("für mich selbst")') if for_myself_option: await for_myself_option.click() logger.info("[GEWOBAG] Selected 'für mich selbst'") form_filled = True except Exception as e: logger.warning(f"[GEWOBAG] Could not select 'für wen' dropdown: {e}") # Mobilfunknummer (Mobile phone) - REQUIRED try: mobile_phone = os.environ.get("FORM_PHONE", "") mobile_input = await iframe_page.query_selector('#formly_17_input_$$_telephone_number_$$_0') if mobile_input: await mobile_input.fill(mobile_phone) logger.info(f"[GEWOBAG] Filled Mobilfunknummer: {mobile_phone}") form_filled = True except Exception as e: logger.warning(f"[GEWOBAG] Could not fill Mobilfunknummer: {e}") # WBS (Wohnberechtigungsschein) - Click "Ja" radio button try: wbs_ja = await iframe_page.query_selector('input[type="radio"][id*="wbs_available"][id*="-Ja"]') if wbs_ja: await wbs_ja.click() logger.info("[GEWOBAG] Selected WBS: Ja") form_filled = True except Exception as e: logger.warning(f"[GEWOBAG] Could not select WBS: {e}") # Privacy checkbox (Main Datenschutzbestimmungen) - REQUIRED try: privacy_checkbox = await iframe_page.query_selector('#formly_20_checkbox_gewobag_datenschutzhinweis_bestaetigt_0') if privacy_checkbox: await privacy_checkbox.check() logger.info("[GEWOBAG] Checked privacy checkbox (Datenschutzbestimmungen)") form_filled = True except Exception as e: logger.warning(f"[GEWOBAG] Could not check privacy checkbox: {e}") await asyncio.sleep(1) # Screenshot after filling screenshot_path = DATA_DIR / f"gewobag_filled_{listing['id']}.png" await iframe_page.screenshot(path=str(screenshot_path), full_page=True) logger.info("[GEWOBAG] Saved filled form screenshot") # Try to submit if form_filled: try: submit_selectors = [ 'button[type="submit"]', 'button:has-text("Absenden")', 'button:has-text("Senden")', 'button:has-text("Anfrage")', '.btn-primary', ] submit_btn = None for selector in submit_selectors: submit_btn = await iframe_page.query_selector(selector) if submit_btn and await submit_btn.is_visible(): logger.info(f"[GEWOBAG] Found submit button: {selector}") break submit_btn = None if submit_btn: await submit_btn.click() logger.info("[GEWOBAG] Clicked submit button") await asyncio.sleep(3) # Screenshot after submission screenshot_path = DATA_DIR / f"gewobag_submitted_{listing['id']}.png" await iframe_page.screenshot(path=str(screenshot_path), full_page=True) logger.info("[GEWOBAG] Saved submission screenshot") # Check page content for errors or confirmation page_content = await iframe_page.content() # Check for validation errors first if "Es wurden nicht alle Felder korrekt befüllt" in page_content or "nicht alle Felder korrekt" in page_content: result["success"] = False result["message"] = "Form validation error: Not all fields filled correctly" logger.warning("[GEWOBAG] Form validation error detected") # Check for success confirmation elif any(phrase in page_content for phrase in [ "Vielen Dank", "Ihre Anfrage wurde", "erfolgreich", "Bestätigung", "Danke für Ihre Bewerbung", "Bewerbung erhalten" ]): result["success"] = True result["message"] = "Application submitted successfully - confirmation detected" logger.info("[GEWOBAG] Success confirmation detected") else: # No confirmation found - mark as failed result["success"] = False result["message"] = "Submitted but no confirmation message found - check screenshot" logger.warning("[GEWOBAG] No confirmation message found after submission") else: result["success"] = False result["message"] = "Form filled but submit button not found" logger.warning("[GEWOBAG] Submit button not found") except Exception as e: result["success"] = False result["message"] = f"Submit error: {str(e)}" logger.warning(f"[GEWOBAG] Submit error: {e}") else: result["success"] = False result["message"] = "No form fields found in Wohnungshelden" logger.warning("[GEWOBAG] Could not find form fields") finally: await iframe_page.close() else: result["success"] = False result["message"] = "No Wohnungshelden iframe found" logger.warning("[GEWOBAG] No Wohnungshelden iframe found") screenshot_path = DATA_DIR / f"gewobag_nobtn_{listing['id']}.png" await page.screenshot(path=str(screenshot_path)) except Exception as e: result["message"] = f"Error during application: {e}" logger.error(f"[GEWOBAG] Application error: {e}") finally: await page.close() return result