315 lines
No EOL
17 KiB
Python
315 lines
No EOL
17 KiB
Python
from .base_handler import BaseHandler
|
|
import logging
|
|
import asyncio
|
|
import os
|
|
from pathlib import Path
|
|
|
|
logger = logging.getLogger(__name__)
|
|
DATA_DIR = Path("data/gewobag")
|
|
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
|
class GewobagHandler(BaseHandler):
|
|
def __init__(self, browser_context):
|
|
self.context = browser_context
|
|
|
|
async def apply(self, listing: dict, result: dict) -> dict:
|
|
page = await self.context.new_page()
|
|
try:
|
|
logger.info(f"[GEWOBAG] Opening page: {listing['link']}")
|
|
response = await page.goto(listing["link"], wait_until="networkidle")
|
|
logger.info("[GEWOBAG] Page loaded")
|
|
await asyncio.sleep(2)
|
|
|
|
# Detect 404 by status, page title, or "nicht gefunden" message
|
|
status = response.status if response else None
|
|
page_title = await page.title()
|
|
page_content = await page.content()
|
|
is_404 = (
|
|
status == 404 or
|
|
(page_title and "404" in page_title) or
|
|
(page_title and "nicht gefunden" in page_title.lower()) or
|
|
("Mietangebot nicht gefunden" in page_content)
|
|
)
|
|
if is_404:
|
|
logger.warning(f"[GEWOBAG] Listing is down (404 or unavailable): {listing['link']}")
|
|
result["success"] = False
|
|
result["message"] = "Listing is no longer available (404). Application impossible. Will not retry."
|
|
result["deactivated"] = True
|
|
return result
|
|
|
|
# Dismiss cookie banner
|
|
try:
|
|
cookie_btn = await page.query_selector('#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll, button:has-text("Alle akzeptieren")')
|
|
if cookie_btn and await cookie_btn.is_visible():
|
|
await cookie_btn.click()
|
|
logger.info("[GEWOBAG] Dismissed cookie banner")
|
|
await asyncio.sleep(1)
|
|
except:
|
|
pass
|
|
|
|
# Gewobag uses Wohnungshelden iframe directly on the page
|
|
logger.info("[GEWOBAG] Looking for Wohnungshelden iframe...")
|
|
iframe_element = await page.query_selector('iframe[src*="wohnungshelden.de"]')
|
|
|
|
if iframe_element:
|
|
iframe_url = await iframe_element.get_attribute('src')
|
|
logger.info(f"[GEWOBAG] Found Wohnungshelden iframe: {iframe_url}")
|
|
|
|
# Navigate to the iframe URL directly in a new page
|
|
iframe_page = await self.context.new_page()
|
|
try:
|
|
await iframe_page.goto(iframe_url, wait_until="networkidle")
|
|
await asyncio.sleep(2)
|
|
logger.info("[GEWOBAG] Loaded Wohnungshelden application page")
|
|
|
|
# Take screenshot
|
|
screenshot_path = DATA_DIR / f"gewobag_wohnungshelden_{listing['id']}.png"
|
|
await iframe_page.screenshot(path=str(screenshot_path), full_page=True)
|
|
logger.info("[GEWOBAG] Saved Wohnungshelden screenshot")
|
|
|
|
# Save HTML for debugging
|
|
try:
|
|
html_content = await iframe_page.content()
|
|
with open(DATA_DIR / f"gewobag_wohnungshelden_{listing['id']}.html", "w", encoding="utf-8") as f:
|
|
f.write(html_content)
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not save HTML: {e}")
|
|
|
|
# Fill out Wohnungshelden form
|
|
form_filled = False
|
|
|
|
# Anrede (Salutation) - ng-select dropdown
|
|
try:
|
|
salutation_dropdown = await iframe_page.query_selector('#salutation-dropdown, ng-select[id*="salutation"]')
|
|
if salutation_dropdown:
|
|
await salutation_dropdown.click()
|
|
await asyncio.sleep(0.5)
|
|
anrede_option = await iframe_page.query_selector(f'.ng-option:has-text("{os.environ.get("FORM_ANREDE", "Herr")}")')
|
|
if anrede_option:
|
|
await anrede_option.click()
|
|
logger.info(f"[GEWOBAG] Selected Anrede: {os.environ.get('FORM_ANREDE', 'Herr')}")
|
|
form_filled = True
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not set Anrede: {e}")
|
|
|
|
# Vorname (First name)
|
|
try:
|
|
vorname_field = await iframe_page.query_selector('#firstName')
|
|
if vorname_field:
|
|
await vorname_field.fill(os.environ.get("FORM_VORNAME", ""))
|
|
logger.info(f"[GEWOBAG] Filled Vorname: {os.environ.get('FORM_VORNAME', '')}")
|
|
form_filled = True
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not fill Vorname: {e}")
|
|
|
|
# Nachname (Last name)
|
|
try:
|
|
nachname_field = await iframe_page.query_selector('#lastName')
|
|
if nachname_field:
|
|
await nachname_field.fill(os.environ.get("FORM_NACHNAME", ""))
|
|
logger.info(f"[GEWOBAG] Filled Nachname: {os.environ.get('FORM_NACHNAME', '')}")
|
|
form_filled = True
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not fill Nachname: {e}")
|
|
|
|
# E-Mail
|
|
try:
|
|
email_field = await iframe_page.query_selector('#email')
|
|
if email_field:
|
|
await email_field.fill(os.environ.get("FORM_EMAIL", ""))
|
|
logger.info(f"[GEWOBAG] Filled E-Mail: {os.environ.get('FORM_EMAIL', '')}")
|
|
form_filled = True
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not fill E-Mail: {e}")
|
|
|
|
# Telefonnummer
|
|
try:
|
|
tel_field = await iframe_page.query_selector('#phone-number, input[id*="telefonnummer"], input[id*="phone"]')
|
|
if tel_field:
|
|
await tel_field.fill(os.environ.get("FORM_PHONE", ""))
|
|
logger.info(f"[GEWOBAG] Filled Telefon: {os.environ.get('FORM_PHONE', '')}")
|
|
form_filled = True
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not fill Telefon: {e}")
|
|
|
|
# Straße (Street) - formcontrolname="street"
|
|
try:
|
|
strasse_field = await iframe_page.query_selector('#street, input[formcontrolname="street"]')
|
|
if strasse_field:
|
|
await strasse_field.fill(os.environ.get("FORM_STRASSE", ""))
|
|
logger.info(f"[GEWOBAG] Filled Straße: {os.environ.get('FORM_STRASSE', '')}")
|
|
form_filled = True
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not fill Straße: {e}")
|
|
|
|
# Hausnummer (House number) - formcontrolname="houseNumber"
|
|
try:
|
|
hausnr_field = await iframe_page.query_selector('input[formcontrolname="houseNumber"]')
|
|
if hausnr_field:
|
|
await hausnr_field.fill(os.environ.get("FORM_HAUSNUMMER", ""))
|
|
logger.info(f"[GEWOBAG] Filled Hausnummer: {os.environ.get('FORM_HAUSNUMMER', '')}")
|
|
form_filled = True
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not fill Hausnummer: {e}")
|
|
|
|
# PLZ (Postal code) - formcontrolname="zipCode"
|
|
try:
|
|
plz_field = await iframe_page.query_selector('input[formcontrolname="zipCode"]')
|
|
if plz_field:
|
|
await plz_field.fill(os.environ.get("FORM_PLZ", ""))
|
|
logger.info(f"[GEWOBAG] Filled PLZ: {os.environ.get('FORM_PLZ', '')}")
|
|
form_filled = True
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not fill PLZ: {e}")
|
|
|
|
# Ort (City) - formcontrolname="city"
|
|
try:
|
|
ort_field = await iframe_page.query_selector('input[formcontrolname="city"]')
|
|
if ort_field:
|
|
await ort_field.fill(os.environ.get("FORM_ORT", ""))
|
|
logger.info(f"[GEWOBAG] Filled Ort: {os.environ.get('FORM_ORT', '')}")
|
|
form_filled = True
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not fill Ort: {e}")
|
|
|
|
# Anzahl der einziehenden Erwachsenen
|
|
try:
|
|
anzahl_erwachsene = os.environ.get("FORM_ADULTS", "1")
|
|
adults_input = await iframe_page.query_selector('#formly_3_input_gewobag_anzahl_erwachsene_0')
|
|
if adults_input:
|
|
await adults_input.fill(anzahl_erwachsene)
|
|
logger.info(f"[GEWOBAG] Filled Anzahl Erwachsene: {anzahl_erwachsene}")
|
|
form_filled = True
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not fill Anzahl Erwachsene: {e}")
|
|
|
|
# Anzahl der einziehenden Kinder
|
|
try:
|
|
anzahl_kinder = os.environ.get("FORM_CHILDREN", "0")
|
|
children_input = await iframe_page.query_selector('#formly_3_input_gewobag_anzahl_kinder_1')
|
|
if children_input:
|
|
await children_input.fill(anzahl_kinder)
|
|
logger.info(f"[GEWOBAG] Filled Anzahl Kinder: {anzahl_kinder}")
|
|
form_filled = True
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not fill Anzahl Kinder: {e}")
|
|
|
|
# WBS (Wohnberechtigungsschein) - Click "Ja" radio button
|
|
try:
|
|
wbs_ja = await iframe_page.query_selector('input[type="radio"][id*="wbs_available"][id*="-Ja"]')
|
|
if wbs_ja:
|
|
await wbs_ja.click()
|
|
logger.info("[GEWOBAG] Selected WBS: Ja")
|
|
form_filled = True
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not select WBS: {e}")
|
|
|
|
# Privacy checkbox 1 (WBS data consent)
|
|
try:
|
|
privacy_checkbox_1 = await iframe_page.query_selector('#formly_20_checkbox_gewobag_datenschutzhinweis_iv0027_bestaetigt_0')
|
|
if privacy_checkbox_1:
|
|
await privacy_checkbox_1.check()
|
|
logger.info("[GEWOBAG] Checked privacy checkbox 1 (WBS data consent)")
|
|
form_filled = True
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not check privacy checkbox 1: {e}")
|
|
|
|
# Privacy checkbox 2 (Main Datenschutzbestimmungen)
|
|
try:
|
|
privacy_checkbox_2 = await iframe_page.query_selector('#formly_21_checkbox_gewobag_datenschutzhinweis_bestaetigt_0')
|
|
if privacy_checkbox_2:
|
|
await privacy_checkbox_2.check()
|
|
logger.info("[GEWOBAG] Checked privacy checkbox 2 (Datenschutzbestimmungen)")
|
|
form_filled = True
|
|
except Exception as e:
|
|
logger.warning(f"[GEWOBAG] Could not check privacy checkbox 2: {e}")
|
|
|
|
await asyncio.sleep(1)
|
|
|
|
# Screenshot after filling
|
|
screenshot_path = DATA_DIR / f"gewobag_filled_{listing['id']}.png"
|
|
await iframe_page.screenshot(path=str(screenshot_path), full_page=True)
|
|
logger.info("[GEWOBAG] Saved filled form screenshot")
|
|
|
|
# Try to submit
|
|
if form_filled:
|
|
try:
|
|
submit_selectors = [
|
|
'button[type="submit"]',
|
|
'button:has-text("Absenden")',
|
|
'button:has-text("Senden")',
|
|
'button:has-text("Anfrage")',
|
|
'.btn-primary',
|
|
]
|
|
|
|
submit_btn = None
|
|
for selector in submit_selectors:
|
|
submit_btn = await iframe_page.query_selector(selector)
|
|
if submit_btn and await submit_btn.is_visible():
|
|
logger.info(f"[GEWOBAG] Found submit button: {selector}")
|
|
break
|
|
submit_btn = None
|
|
|
|
if submit_btn:
|
|
await submit_btn.click()
|
|
logger.info("[GEWOBAG] Clicked submit button")
|
|
await asyncio.sleep(3)
|
|
|
|
# Screenshot after submission
|
|
screenshot_path = DATA_DIR / f"gewobag_submitted_{listing['id']}.png"
|
|
await iframe_page.screenshot(path=str(screenshot_path), full_page=True)
|
|
logger.info("[GEWOBAG] Saved submission screenshot")
|
|
|
|
# Check page content for errors or confirmation
|
|
page_content = await iframe_page.content()
|
|
|
|
# Check for validation errors first
|
|
if "Es wurden nicht alle Felder korrekt befüllt" in page_content or "nicht alle Felder korrekt" in page_content:
|
|
result["success"] = False
|
|
result["message"] = "Form validation error: Not all fields filled correctly"
|
|
logger.warning("[GEWOBAG] Form validation error detected")
|
|
# Check for success confirmation
|
|
elif any(phrase in page_content for phrase in [
|
|
"Vielen Dank",
|
|
"Ihre Anfrage wurde",
|
|
"erfolgreich",
|
|
"Bestätigung",
|
|
"Danke für Ihre Bewerbung",
|
|
"Bewerbung erhalten"
|
|
]):
|
|
result["success"] = True
|
|
result["message"] = "Application submitted successfully - confirmation detected"
|
|
logger.info("[GEWOBAG] Success confirmation detected")
|
|
else:
|
|
# No confirmation found - mark as failed
|
|
result["success"] = False
|
|
result["message"] = "Submitted but no confirmation message found - check screenshot"
|
|
logger.warning("[GEWOBAG] No confirmation message found after submission")
|
|
else:
|
|
result["success"] = False
|
|
result["message"] = "Form filled but submit button not found"
|
|
logger.warning("[GEWOBAG] Submit button not found")
|
|
except Exception as e:
|
|
result["success"] = False
|
|
result["message"] = f"Submit error: {str(e)}"
|
|
logger.warning(f"[GEWOBAG] Submit error: {e}")
|
|
else:
|
|
result["success"] = False
|
|
result["message"] = "No form fields found in Wohnungshelden"
|
|
logger.warning("[GEWOBAG] Could not find form fields")
|
|
finally:
|
|
await iframe_page.close()
|
|
else:
|
|
result["success"] = False
|
|
result["message"] = "No Wohnungshelden iframe found"
|
|
logger.warning("[GEWOBAG] No Wohnungshelden iframe found")
|
|
screenshot_path = DATA_DIR / f"gewobag_nobtn_{listing['id']}.png"
|
|
await page.screenshot(path=str(screenshot_path))
|
|
|
|
except Exception as e:
|
|
result["message"] = f"Error during application: {e}"
|
|
logger.error(f"[GEWOBAG] Application error: {e}")
|
|
finally:
|
|
await page.close()
|
|
|
|
return result |