341 lines
No EOL
19 KiB
Python
341 lines
No EOL
19 KiB
Python
from .base_handler import BaseHandler
|
|
import logging
|
|
import asyncio
|
|
import os
|
|
from pathlib import Path
|
|
|
|
logger = logging.getLogger(__name__)
|
|
DATA_DIR = Path("data/gewobag")
|
|
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
|
class GewobagHandler(BaseHandler):
|
|
def __init__(self, browser_context):
|
|
self.context = browser_context
|
|
|
|
async def apply(self, listing: dict, result: dict) -> dict:
|
|
page = await self.context.new_page()
|
|
try:
|
|
logger.info(f"[GEWOBAG] Opening page: {listing['link']}")
|
|
response = await page.goto(listing["link"], wait_until="networkidle")
|
|
logger.info("[GEWOBAG] Page loaded")
|
|
await asyncio.sleep(2)
|
|
|
|
# Detect 404 by status, page title, or "nicht gefunden" message
|
|
status = response.status if response else None
|
|
page_title = await page.title()
|
|
page_content = await page.content()
|
|
is_404 = (
|
|
status == 404 or
|
|
(page_title and "404" in page_title) or
|
|
(page_title and "nicht gefunden" in page_title.lower()) or
|
|
("Mietangebot nicht gefunden" in page_content) or
|
|
("Bewerbungsphase abgeschlossen" in page_content)
|
|
)
|
|
if is_404:
|
|
logger.warning(f"[GEWOBAG] Listing is down (404, unavailable, or application phase closed): {listing['link']}")
|
|
result["success"] = False
|
|
result["message"] = "Listing no longer available or application phase closed"
|
|
result["deactivated"] = True
|
|
return result
|
|
|
|
# Dismiss cookie banner
|
|
try:
|
|
cookie_btn = await page.query_selector('#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll, button:has-text("Alle akzeptieren")')
|
|
if cookie_btn and await cookie_btn.is_visible():
|
|
await cookie_btn.click()
|
|
logger.info("[GEWOBAG] Dismissed cookie banner")
|
|
await asyncio.sleep(1)
|
|
except:
|
|
pass
|
|
|
|
# Gewobag uses Wohnungshelden iframe directly on the page
|
|
logger.info("[GEWOBAG] Looking for Wohnungshelden iframe...")
|
|
iframe_element = await page.query_selector('iframe[src*="wohnungshelden.de"]')
|
|
|
|
if iframe_element:
|
|
iframe_url = await iframe_element.get_attribute('src')
|
|
logger.info(f"[GEWOBAG] Found Wohnungshelden iframe: {iframe_url}")
|
|
|
|
# Navigate to the iframe URL directly in a new page
|
|
iframe_page = await self.context.new_page()
|
|
try:
|
|
await iframe_page.goto(iframe_url, wait_until="networkidle")
|
|
await asyncio.sleep(2)
|
|
logger.info("[GEWOBAG] Loaded Wohnungshelden application page")
|
|
|
|
# Take screenshot
|
|
screenshot_path = DATA_DIR / f"gewobag_wohnungshelden_{listing['id']}.png"
|
|
await iframe_page.screenshot(path=str(screenshot_path), full_page=True)
|
|
logger.info("[GEWOBAG] Saved Wohnungshelden screenshot")
|
|
|
|
# Save HTML for debugging
|
|
try:
|
|
html_content = await iframe_page.content()
|
|
with open(DATA_DIR / f"gewobag_wohnungshelden_{listing['id']}.html", "w", encoding="utf-8") as f:
|
|
f.write(html_content)
|
|
|
|
# Check for Wohnungshelden error messages
|
|
if "Fehler beim Laden" in html_content or "Beim Laden der Gesuchsdaten ist ein Fehler aufgetreten" in html_content:
|
|
logger.warning("[GEWOBAG] Wohnungshelden iframe shows error - may be transient, will retry")
|
|
result["success"] = False
|
|
result["message"] = "Wohnungshelden application form error - will retry later"
|
|
await iframe_page.close()
|
|
return result
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not save HTML: {e}")
|
|
|
|
# Fill out Wohnungshelden form
|
|
form_filled = False
|
|
|
|
# Anrede (Salutation) - ng-select dropdown
|
|
try:
|
|
salutation_dropdown = await iframe_page.query_selector('#salutation-dropdown, ng-select[id*="salutation"]')
|
|
if salutation_dropdown:
|
|
await salutation_dropdown.click()
|
|
await asyncio.sleep(0.5)
|
|
anrede_option = await iframe_page.query_selector(f'.ng-option:has-text("{os.environ.get("FORM_ANREDE", "Herr")}")')
|
|
if anrede_option:
|
|
await anrede_option.click()
|
|
logger.info(f"[GEWOBAG] Selected Anrede: {os.environ.get('FORM_ANREDE', 'Herr')}")
|
|
form_filled = True
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not set Anrede: {e}")
|
|
|
|
# Vorname (First name)
|
|
try:
|
|
vorname_field = await iframe_page.query_selector('#firstName')
|
|
if vorname_field:
|
|
await vorname_field.fill(os.environ.get("FORM_VORNAME", ""))
|
|
logger.info(f"[GEWOBAG] Filled Vorname: {os.environ.get('FORM_VORNAME', '')}")
|
|
form_filled = True
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not fill Vorname: {e}")
|
|
|
|
# Nachname (Last name)
|
|
try:
|
|
nachname_field = await iframe_page.query_selector('#lastName')
|
|
if nachname_field:
|
|
await nachname_field.fill(os.environ.get("FORM_NACHNAME", ""))
|
|
logger.info(f"[GEWOBAG] Filled Nachname: {os.environ.get('FORM_NACHNAME', '')}")
|
|
form_filled = True
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not fill Nachname: {e}")
|
|
|
|
# E-Mail
|
|
try:
|
|
email_field = await iframe_page.query_selector('#email')
|
|
if email_field:
|
|
await email_field.fill(os.environ.get("FORM_EMAIL", ""))
|
|
logger.info(f"[GEWOBAG] Filled E-Mail: {os.environ.get('FORM_EMAIL', '')}")
|
|
form_filled = True
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not fill E-Mail: {e}")
|
|
|
|
# Telefonnummer
|
|
try:
|
|
tel_field = await iframe_page.query_selector('#phone-number, input[id*="telefonnummer"], input[id*="phone"]')
|
|
if tel_field:
|
|
await tel_field.fill(os.environ.get("FORM_PHONE", ""))
|
|
logger.info(f"[GEWOBAG] Filled Telefon: {os.environ.get('FORM_PHONE', '')}")
|
|
form_filled = True
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not fill Telefon: {e}")
|
|
|
|
# Straße (Street) - formcontrolname="street"
|
|
try:
|
|
strasse_field = await iframe_page.query_selector('#street, input[formcontrolname="street"]')
|
|
if strasse_field:
|
|
await strasse_field.fill(os.environ.get("FORM_STRASSE", ""))
|
|
logger.info(f"[GEWOBAG] Filled Straße: {os.environ.get('FORM_STRASSE', '')}")
|
|
form_filled = True
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not fill Straße: {e}")
|
|
|
|
# Hausnummer (House number) - formcontrolname="houseNumber"
|
|
try:
|
|
hausnr_field = await iframe_page.query_selector('input[formcontrolname="houseNumber"]')
|
|
if hausnr_field:
|
|
await hausnr_field.fill(os.environ.get("FORM_HAUSNUMMER", ""))
|
|
logger.info(f"[GEWOBAG] Filled Hausnummer: {os.environ.get('FORM_HAUSNUMMER', '')}")
|
|
form_filled = True
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not fill Hausnummer: {e}")
|
|
|
|
# PLZ (Postal code) - formcontrolname="zipCode"
|
|
try:
|
|
plz_field = await iframe_page.query_selector('input[formcontrolname="zipCode"]')
|
|
if plz_field:
|
|
await plz_field.fill(os.environ.get("FORM_PLZ", ""))
|
|
logger.info(f"[GEWOBAG] Filled PLZ: {os.environ.get('FORM_PLZ', '')}")
|
|
form_filled = True
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not fill PLZ: {e}")
|
|
|
|
# Ort (City) - formcontrolname="city"
|
|
try:
|
|
ort_field = await iframe_page.query_selector('input[formcontrolname="city"]')
|
|
if ort_field:
|
|
await ort_field.fill(os.environ.get("FORM_ORT", ""))
|
|
logger.info(f"[GEWOBAG] Filled Ort: {os.environ.get('FORM_ORT', '')}")
|
|
form_filled = True
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not fill Ort: {e}")
|
|
|
|
# Gesamtzahl der einziehenden Personen (Erwachsene + Kinder) - REQUIRED
|
|
try:
|
|
anzahl_erwachsene = int(os.environ.get("FORM_ADULTS", "1"))
|
|
anzahl_kinder = int(os.environ.get("FORM_CHILDREN", "0"))
|
|
total_persons = str(anzahl_erwachsene + anzahl_kinder)
|
|
total_input = await iframe_page.query_selector('#formly_2_input_gewobag_gesamtzahl_der_einziehenden_personen_erwachsene_und_kinder_0')
|
|
if total_input:
|
|
await total_input.fill(total_persons)
|
|
logger.info(f"[GEWOBAG] Filled Gesamtzahl Personen: {total_persons}")
|
|
form_filled = True
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not fill Gesamtzahl Personen: {e}")
|
|
|
|
# "Für wen wird die Wohnungsanfrage gestellt?" dropdown - REQUIRED
|
|
# Select "für mich selbst" (for myself)
|
|
try:
|
|
dropdown_input = await iframe_page.query_selector('#formly_9_select_gewobag_fuer_wen_wird_die_wohnungsanfrage_gestellt_0')
|
|
if dropdown_input:
|
|
await dropdown_input.click()
|
|
await iframe_page.wait_for_timeout(300)
|
|
# Select the first option "für mich selbst"
|
|
for_myself_option = await iframe_page.query_selector('.ng-option:has-text("für mich selbst")')
|
|
if for_myself_option:
|
|
await for_myself_option.click()
|
|
logger.info("[GEWOBAG] Selected 'für mich selbst'")
|
|
await asyncio.sleep(1) # Wait for conditional fields to appear
|
|
form_filled = True
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not select 'für wen' dropdown: {e}")
|
|
|
|
# Mobilfunknummer (Mobile phone) - REQUIRED (appears after dropdown selection)
|
|
try:
|
|
mobile_phone = os.environ.get("FORM_PHONE", "")
|
|
if mobile_phone:
|
|
# Wait for the field to appear - need to escape $$ in the selector
|
|
await iframe_page.wait_for_selector('#formly_17_input_\\$\\$_telephone_number_\\$\\$_0', timeout=5000, state='visible')
|
|
mobile_input = await iframe_page.query_selector('#formly_17_input_\\$\\$_telephone_number_\\$\\$_0')
|
|
if mobile_input:
|
|
await mobile_input.scroll_into_view_if_needed()
|
|
await mobile_input.fill(mobile_phone)
|
|
logger.info(f"[GEWOBAG] Filled Mobilfunknummer: {mobile_phone}")
|
|
form_filled = True
|
|
else:
|
|
logger.warning("[GEWOBAG] Mobilfunknummer field not found")
|
|
else:
|
|
logger.warning("[GEWOBAG] FORM_PHONE environment variable is empty")
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not fill Mobilfunknummer: {e}")
|
|
|
|
# WBS (Wohnberechtigungsschein) - Click "Ja" radio button
|
|
try:
|
|
wbs_ja = await iframe_page.query_selector('input[type="radio"][id*="wbs_available"][id*="-Ja"]')
|
|
if wbs_ja:
|
|
await wbs_ja.click()
|
|
logger.info("[GEWOBAG] Selected WBS: Ja")
|
|
form_filled = True
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not select WBS: {e}")
|
|
|
|
# Privacy checkbox (Main Datenschutzbestimmungen) - REQUIRED
|
|
try:
|
|
privacy_checkbox = await iframe_page.query_selector('#formly_20_checkbox_gewobag_datenschutzhinweis_bestaetigt_0')
|
|
if privacy_checkbox:
|
|
await privacy_checkbox.check()
|
|
logger.info("[GEWOBAG] Checked privacy checkbox (Datenschutzbestimmungen)")
|
|
form_filled = True
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not check privacy checkbox: {e}")
|
|
|
|
await asyncio.sleep(1)
|
|
|
|
# Screenshot after filling
|
|
screenshot_path = DATA_DIR / f"gewobag_filled_{listing['id']}.png"
|
|
await iframe_page.screenshot(path=str(screenshot_path), full_page=True)
|
|
logger.info("[GEWOBAG] Saved filled form screenshot")
|
|
|
|
# Try to submit
|
|
if form_filled:
|
|
try:
|
|
submit_selectors = [
|
|
'button[type="submit"]',
|
|
'button:has-text("Absenden")',
|
|
'button:has-text("Senden")',
|
|
'button:has-text("Anfrage")',
|
|
'.btn-primary',
|
|
]
|
|
|
|
submit_btn = None
|
|
for selector in submit_selectors:
|
|
submit_btn = await iframe_page.query_selector(selector)
|
|
if submit_btn and await submit_btn.is_visible():
|
|
logger.info(f"[GEWOBAG] Found submit button: {selector}")
|
|
break
|
|
submit_btn = None
|
|
|
|
if submit_btn:
|
|
await submit_btn.click()
|
|
logger.info("[GEWOBAG] Clicked submit button")
|
|
await asyncio.sleep(3)
|
|
|
|
# Screenshot after submission
|
|
screenshot_path = DATA_DIR / f"gewobag_submitted_{listing['id']}.png"
|
|
await iframe_page.screenshot(path=str(screenshot_path), full_page=True)
|
|
logger.info("[GEWOBAG] Saved submission screenshot")
|
|
|
|
# Check page content for errors or confirmation
|
|
page_content = await iframe_page.content()
|
|
|
|
# Check for validation errors first
|
|
if "Es wurden nicht alle Felder korrekt befüllt" in page_content or "nicht alle Felder korrekt" in page_content:
|
|
result["success"] = False
|
|
result["message"] = "Form validation error: Not all fields filled correctly"
|
|
logger.warning("[GEWOBAG] Form validation error detected")
|
|
# Check for success confirmation
|
|
elif any(phrase in page_content for phrase in [
|
|
"Vielen Dank",
|
|
"Ihre Anfrage wurde",
|
|
"erfolgreich",
|
|
"Bestätigung",
|
|
"Danke für Ihre Bewerbung",
|
|
"Bewerbung erhalten"
|
|
]):
|
|
result["success"] = True
|
|
result["message"] = "Application submitted successfully - confirmation detected"
|
|
logger.info("[GEWOBAG] Success confirmation detected")
|
|
else:
|
|
# No confirmation found - mark as failed
|
|
result["success"] = False
|
|
result["message"] = "Submitted but no confirmation message found - check screenshot"
|
|
logger.warning("[GEWOBAG] No confirmation message found after submission")
|
|
else:
|
|
result["success"] = False
|
|
result["message"] = "Form filled but submit button not found"
|
|
logger.warning("[GEWOBAG] Submit button not found")
|
|
except Exception as e:
|
|
result["success"] = False
|
|
result["message"] = f"Submit error: {str(e)}"
|
|
logger.warning(f"[GEWOBAG] Submit error: {e}")
|
|
else:
|
|
result["success"] = False
|
|
result["message"] = "No form fields found in Wohnungshelden"
|
|
logger.warning("[GEWOBAG] Could not find form fields")
|
|
finally:
|
|
await iframe_page.close()
|
|
else:
|
|
result["success"] = False
|
|
result["message"] = "No Wohnungshelden iframe found"
|
|
logger.warning("[GEWOBAG] No Wohnungshelden iframe found")
|
|
screenshot_path = DATA_DIR / f"gewobag_nobtn_{listing['id']}.png"
|
|
await page.screenshot(path=str(screenshot_path))
|
|
|
|
except Exception as e:
|
|
result["message"] = f"Error during application: {e}"
|
|
logger.error(f"[GEWOBAG] Application error: {e}")
|
|
finally:
|
|
await page.close()
|
|
|
|
return result |