Compare commits
No commits in common. "2b16e52a53c1ae6f958dc0e05cd0557e551ee6c4" and "4039fb64b7dc0858ab6dd536e3630eea33f8caa1" have entirely different histories.
2b16e52a53
...
4039fb64b7
1 changed files with 234 additions and 401 deletions
553
monitor.py
553
monitor.py
|
|
@ -450,99 +450,90 @@ class ApplicationHandler:
|
||||||
await apply_btn.click()
|
await apply_btn.click()
|
||||||
await asyncio.sleep(3)
|
await asyncio.sleep(3)
|
||||||
await page.wait_for_load_state("networkidle")
|
await page.wait_for_load_state("networkidle")
|
||||||
logger.info("[HOWOGE] Clicked button, starting multi-step form process...")
|
logger.info("[HOWOGE] Clicked button, waiting for form...")
|
||||||
|
|
||||||
# HOWOGE has a multi-step form (typically 3-4 steps):
|
# Scroll down the page to reveal checkboxes
|
||||||
# Each step has a checkbox that must be clicked, then "Weiter" button
|
await page.evaluate("window.scrollBy(0, 500)")
|
||||||
# Final step has the actual contact form
|
await asyncio.sleep(1)
|
||||||
|
logger.info("[HOWOGE] Scrolled down to reveal checkboxes")
|
||||||
|
|
||||||
max_steps = 6 # safety limit
|
# Screenshot after clicking (before checkboxes)
|
||||||
for step in range(1, max_steps + 1):
|
screenshot_path = DATA_DIR / f"howoge_form_{listing['id']}.png"
|
||||||
logger.info(f"[HOWOGE] Processing step {step}")
|
await page.screenshot(path=str(screenshot_path), full_page=True)
|
||||||
|
logger.info(f"[HOWOGE] Saved form screenshot to {screenshot_path}")
|
||||||
|
|
||||||
# Scroll down to reveal checkboxes
|
# HOWOGE requires clicking checkboxes before the form fields become visible
|
||||||
await page.evaluate("window.scrollBy(0, 300)")
|
# Look for and click all required checkboxes
|
||||||
await asyncio.sleep(0.5)
|
checkboxes_clicked = 0
|
||||||
|
checkbox_selectors = [
|
||||||
|
'input[type="checkbox"]',
|
||||||
|
'.checkbox input',
|
||||||
|
'input[name*="datenschutz" i]',
|
||||||
|
'input[name*="privacy" i]',
|
||||||
|
'input[name*="einwillig" i]',
|
||||||
|
'input[name*="zustimmung" i]',
|
||||||
|
'input[name*="accept" i]',
|
||||||
|
]
|
||||||
|
|
||||||
# Check if we've reached the form (email field is visible)
|
for selector in checkbox_selectors:
|
||||||
email_field = await page.query_selector('input[name*="email" i]')
|
checkboxes = await page.query_selector_all(selector)
|
||||||
if email_field and await email_field.is_visible():
|
|
||||||
logger.info("[HOWOGE] Email field is visible - form is ready!")
|
|
||||||
break
|
|
||||||
|
|
||||||
# Find and click any visible unchecked checkboxes
|
|
||||||
checkboxes = await page.query_selector_all('input[type="checkbox"]')
|
|
||||||
clicked_checkbox = False
|
|
||||||
for checkbox in checkboxes:
|
for checkbox in checkboxes:
|
||||||
try:
|
try:
|
||||||
if await checkbox.is_visible() and not await checkbox.is_checked():
|
if await checkbox.is_visible() and not await checkbox.is_checked():
|
||||||
# Use JavaScript click to avoid viewport issues
|
await checkbox.scroll_into_view_if_needed()
|
||||||
await checkbox.evaluate("el => el.click()")
|
await checkbox.click()
|
||||||
clicked_checkbox = True
|
checkboxes_clicked += 1
|
||||||
logger.info(f"[HOWOGE] Clicked checkbox in step {step}")
|
logger.info(f"[HOWOGE] Clicked checkbox")
|
||||||
await asyncio.sleep(0.5)
|
await asyncio.sleep(0.3)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.debug(f"[HOWOGE] Checkbox click failed: {e}")
|
logger.debug(f"[HOWOGE] Checkbox click failed: {e}")
|
||||||
|
|
||||||
if clicked_checkbox:
|
if checkboxes_clicked > 0:
|
||||||
await asyncio.sleep(1) # Wait for page to update after checkbox
|
logger.info(f"[HOWOGE] Clicked {checkboxes_clicked} checkboxes")
|
||||||
|
await asyncio.sleep(1)
|
||||||
|
|
||||||
# Screenshot this step
|
# Screenshot after clicking checkboxes
|
||||||
screenshot_path = DATA_DIR / f"howoge_step{step}_{listing['id']}.png"
|
screenshot_path = DATA_DIR / f"howoge_after_checkboxes_{listing['id']}.png"
|
||||||
await page.screenshot(path=str(screenshot_path), full_page=True)
|
await page.screenshot(path=str(screenshot_path), full_page=True)
|
||||||
|
logger.info(f"[HOWOGE] Saved after-checkboxes screenshot")
|
||||||
|
|
||||||
# Look for visible "Weiter" button and click it
|
# Scroll back up to find the form fields
|
||||||
weiter_btns = await page.query_selector_all('button:has-text("Weiter")')
|
await page.evaluate("window.scrollTo(0, 0)")
|
||||||
weiter_clicked = False
|
await asyncio.sleep(0.5)
|
||||||
for btn in weiter_btns:
|
|
||||||
try:
|
|
||||||
if await btn.is_visible():
|
|
||||||
await btn.click()
|
|
||||||
weiter_clicked = True
|
|
||||||
logger.info(f"[HOWOGE] Clicked 'Weiter' button in step {step}")
|
|
||||||
await asyncio.sleep(2)
|
|
||||||
await page.wait_for_load_state("networkidle")
|
|
||||||
break
|
|
||||||
except Exception as e:
|
|
||||||
logger.debug(f"[HOWOGE] Weiter click failed: {e}")
|
|
||||||
|
|
||||||
if not weiter_clicked and not clicked_checkbox:
|
# Fill in the contact form
|
||||||
logger.warning(f"[HOWOGE] No action possible in step {step}, breaking")
|
# Look for name fields (Vorname, Nachname)
|
||||||
break
|
vorname_field = await page.query_selector('input[name*="vorname" i], input[name*="firstname" i], input[placeholder*="Vorname" i], input#vorname')
|
||||||
|
nachname_field = await page.query_selector('input[name*="nachname" i], input[name*="lastname" i], input[name*="surname" i], input[placeholder*="Nachname" i], input#nachname')
|
||||||
# Now try to fill the form
|
email_field = await page.query_selector('input[type="email"], input[name*="email" i], input[name*="mail" i]')
|
||||||
logger.info("[HOWOGE] Attempting to fill form fields...")
|
|
||||||
|
|
||||||
# Look for name fields - HOWOGE uses firstName/lastName
|
|
||||||
vorname_field = await page.query_selector('input[name*="firstName" i], input[name*="vorname" i]')
|
|
||||||
nachname_field = await page.query_selector('input[name*="lastName" i], input[name*="nachname" i]')
|
|
||||||
email_field = await page.query_selector('input[type="email"], input[name*="email" i]')
|
|
||||||
|
|
||||||
form_filled = False
|
form_filled = False
|
||||||
if vorname_field and await vorname_field.is_visible():
|
if vorname_field:
|
||||||
|
await vorname_field.scroll_into_view_if_needed()
|
||||||
|
await asyncio.sleep(0.2)
|
||||||
await vorname_field.fill(FORM_VORNAME)
|
await vorname_field.fill(FORM_VORNAME)
|
||||||
logger.info(f"[HOWOGE] Filled Vorname: {FORM_VORNAME}")
|
logger.info(f"[HOWOGE] Filled Vorname: {FORM_VORNAME}")
|
||||||
form_filled = True
|
form_filled = True
|
||||||
else:
|
|
||||||
logger.warning("[HOWOGE] Vorname field not found or not visible")
|
|
||||||
|
|
||||||
if nachname_field and await nachname_field.is_visible():
|
if nachname_field:
|
||||||
|
await nachname_field.scroll_into_view_if_needed()
|
||||||
|
await asyncio.sleep(0.2)
|
||||||
await nachname_field.fill(FORM_NACHNAME)
|
await nachname_field.fill(FORM_NACHNAME)
|
||||||
logger.info(f"[HOWOGE] Filled Nachname: {FORM_NACHNAME}")
|
logger.info(f"[HOWOGE] Filled Nachname: {FORM_NACHNAME}")
|
||||||
form_filled = True
|
form_filled = True
|
||||||
else:
|
|
||||||
logger.warning("[HOWOGE] Nachname field not found or not visible")
|
|
||||||
|
|
||||||
if email_field and await email_field.is_visible():
|
if email_field:
|
||||||
|
await email_field.scroll_into_view_if_needed()
|
||||||
|
await asyncio.sleep(0.2)
|
||||||
await email_field.fill(FORM_EMAIL)
|
await email_field.fill(FORM_EMAIL)
|
||||||
logger.info(f"[HOWOGE] Filled Email: {FORM_EMAIL}")
|
logger.info(f"[HOWOGE] Filled Email: {FORM_EMAIL}")
|
||||||
form_filled = True
|
form_filled = True
|
||||||
else:
|
|
||||||
logger.warning("[HOWOGE] Email field not found or not visible")
|
|
||||||
|
|
||||||
# Also look for phone field
|
# Also look for phone field
|
||||||
phone_field = await page.query_selector('input[type="tel"], input[name*="telefon" i], input[name*="phone" i]')
|
phone_field = await page.query_selector('input[type="tel"], input[name*="telefon" i], input[name*="phone" i]')
|
||||||
if phone_field and await phone_field.is_visible():
|
if phone_field:
|
||||||
|
await phone_field.scroll_into_view_if_needed()
|
||||||
|
await asyncio.sleep(0.2)
|
||||||
await phone_field.fill(FORM_PHONE)
|
await phone_field.fill(FORM_PHONE)
|
||||||
logger.info(f"[HOWOGE] Filled Phone: {FORM_PHONE}")
|
logger.info(f"[HOWOGE] Filled Phone: {FORM_PHONE}")
|
||||||
|
|
||||||
|
|
@ -552,16 +543,9 @@ class ApplicationHandler:
|
||||||
logger.info(f"[HOWOGE] Saved filled form screenshot to {screenshot_path2}")
|
logger.info(f"[HOWOGE] Saved filled form screenshot to {screenshot_path2}")
|
||||||
|
|
||||||
if form_filled:
|
if form_filled:
|
||||||
# Look for submit button - HOWOGE uses "Anfrage senden"
|
# Look for submit button
|
||||||
# Try specific selectors first, then fall back
|
submit_btn = await page.query_selector('button[type="submit"], input[type="submit"], button:has-text("Absenden"), button:has-text("Senden"), button:has-text("Anfrage")')
|
||||||
submit_btn = None
|
if submit_btn and await submit_btn.is_visible():
|
||||||
for selector in ['button:has-text("Anfrage senden")', 'button:has-text("Absenden")', 'button:has-text("Senden")']:
|
|
||||||
btn = await page.query_selector(selector)
|
|
||||||
if btn and await btn.is_visible():
|
|
||||||
submit_btn = btn
|
|
||||||
logger.info(f"[HOWOGE] Found submit button with selector: {selector}")
|
|
||||||
break
|
|
||||||
if submit_btn:
|
|
||||||
logger.info("[HOWOGE] Found submit button, clicking...")
|
logger.info("[HOWOGE] Found submit button, clicking...")
|
||||||
await submit_btn.click()
|
await submit_btn.click()
|
||||||
await asyncio.sleep(3)
|
await asyncio.sleep(3)
|
||||||
|
|
@ -573,7 +557,7 @@ class ApplicationHandler:
|
||||||
logger.info(f"[HOWOGE] Saved post-submit screenshot to {screenshot_path3}")
|
logger.info(f"[HOWOGE] Saved post-submit screenshot to {screenshot_path3}")
|
||||||
|
|
||||||
content = await page.content()
|
content = await page.content()
|
||||||
if "erfolgreich" in content.lower() or "gesendet" in content.lower() or "danke" in content.lower() or "bestätigung" in content.lower():
|
if "erfolgreich" in content.lower() or "gesendet" in content.lower() or "danke" in content.lower():
|
||||||
result["success"] = True
|
result["success"] = True
|
||||||
result["message"] = "Application submitted successfully"
|
result["message"] = "Application submitted successfully"
|
||||||
logger.info("[HOWOGE] Success! Confirmation message detected")
|
logger.info("[HOWOGE] Success! Confirmation message detected")
|
||||||
|
|
@ -587,8 +571,8 @@ class ApplicationHandler:
|
||||||
logger.warning("[HOWOGE] Could not find submit button")
|
logger.warning("[HOWOGE] Could not find submit button")
|
||||||
else:
|
else:
|
||||||
result["success"] = False
|
result["success"] = False
|
||||||
result["message"] = "Could not find form fields to fill after navigating steps"
|
result["message"] = "Could not find form fields to fill"
|
||||||
logger.warning("[HOWOGE] No form fields found after multi-step navigation")
|
logger.warning("[HOWOGE] No form fields found")
|
||||||
else:
|
else:
|
||||||
result["message"] = "No application button found"
|
result["message"] = "No application button found"
|
||||||
logger.warning("[HOWOGE] Could not find 'Besichtigung vereinbaren' button")
|
logger.warning("[HOWOGE] Could not find 'Besichtigung vereinbaren' button")
|
||||||
|
|
@ -613,11 +597,6 @@ class ApplicationHandler:
|
||||||
return result
|
return result
|
||||||
|
|
||||||
async def _apply_gewobag(self, listing: dict, result: dict) -> dict:
|
async def _apply_gewobag(self, listing: dict, result: dict) -> dict:
|
||||||
"""
|
|
||||||
Gewobag uses Wohnungshelden (app.wohnungshelden.de) for their application system.
|
|
||||||
The application form is embedded in an iframe on the listing page.
|
|
||||||
We navigate directly to the iframe URL to fill the form.
|
|
||||||
"""
|
|
||||||
page = await self.context.new_page()
|
page = await self.context.new_page()
|
||||||
try:
|
try:
|
||||||
logger.info(f"[GEWOBAG] Opening page: {listing['link']}")
|
logger.info(f"[GEWOBAG] Opening page: {listing['link']}")
|
||||||
|
|
@ -633,153 +612,25 @@ class ApplicationHandler:
|
||||||
await asyncio.sleep(1)
|
await asyncio.sleep(1)
|
||||||
except: pass
|
except: pass
|
||||||
|
|
||||||
# Gewobag has Wohnungshelden iframe directly on the page
|
logger.info("[GEWOBAG] Looking for application button...")
|
||||||
logger.info("[GEWOBAG] Looking for Wohnungshelden iframe...")
|
apply_btn = await page.query_selector('a:has-text("Kontakt"), button:has-text("Anfrage"), a.btn:has-text("Anfragen")')
|
||||||
iframe_element = await page.query_selector('iframe[src*="wohnungshelden.de"]')
|
if apply_btn and await apply_btn.is_visible():
|
||||||
|
logger.info("[GEWOBAG] Found application button, clicking...")
|
||||||
if iframe_element:
|
await apply_btn.click()
|
||||||
iframe_url = await iframe_element.get_attribute('src')
|
|
||||||
logger.info(f"[GEWOBAG] Found Wohnungshelden iframe: {iframe_url}")
|
|
||||||
|
|
||||||
# Navigate to the iframe URL directly in a new page
|
|
||||||
iframe_page = await self.context.new_page()
|
|
||||||
try:
|
|
||||||
await iframe_page.goto(iframe_url, wait_until="networkidle")
|
|
||||||
await asyncio.sleep(2)
|
await asyncio.sleep(2)
|
||||||
logger.info("[GEWOBAG] Loaded Wohnungshelden application page")
|
|
||||||
|
|
||||||
# Take screenshot
|
screenshot_path = DATA_DIR / f"gewobag_{listing['id']}.png"
|
||||||
screenshot_path = DATA_DIR / f"gewobag_wohnungshelden_{listing['id']}.png"
|
await page.screenshot(path=str(screenshot_path))
|
||||||
await iframe_page.screenshot(path=str(screenshot_path), full_page=True)
|
logger.info(f"[GEWOBAG] Saved screenshot to {screenshot_path}")
|
||||||
logger.info(f"[GEWOBAG] Saved Wohnungshelden screenshot")
|
|
||||||
|
|
||||||
# Fill out Wohnungshelden form (same fields as Degewo)
|
result["success"] = False
|
||||||
form_filled = False
|
result["message"] = "Application page opened but not submitted (not implemented)"
|
||||||
|
|
||||||
# Anrede (Salutation) - ng-select dropdown
|
|
||||||
try:
|
|
||||||
salutation_dropdown = await iframe_page.query_selector('#salutation-dropdown, ng-select[id*="salutation"]')
|
|
||||||
if salutation_dropdown:
|
|
||||||
await salutation_dropdown.click()
|
|
||||||
await asyncio.sleep(0.5)
|
|
||||||
anrede_option = await iframe_page.query_selector(f'.ng-option:has-text("{FORM_ANREDE}")')
|
|
||||||
if anrede_option:
|
|
||||||
await anrede_option.click()
|
|
||||||
logger.info(f"[GEWOBAG] Selected Anrede: {FORM_ANREDE}")
|
|
||||||
form_filled = True
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning(f"[GEWOBAG] Could not set Anrede: {e}")
|
|
||||||
|
|
||||||
# Vorname (First name)
|
|
||||||
try:
|
|
||||||
vorname_field = await iframe_page.query_selector('#firstName')
|
|
||||||
if vorname_field:
|
|
||||||
await vorname_field.fill(FORM_VORNAME)
|
|
||||||
logger.info(f"[GEWOBAG] Filled Vorname: {FORM_VORNAME}")
|
|
||||||
form_filled = True
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning(f"[GEWOBAG] Could not fill Vorname: {e}")
|
|
||||||
|
|
||||||
# Nachname (Last name)
|
|
||||||
try:
|
|
||||||
nachname_field = await iframe_page.query_selector('#lastName')
|
|
||||||
if nachname_field:
|
|
||||||
await nachname_field.fill(FORM_NACHNAME)
|
|
||||||
logger.info(f"[GEWOBAG] Filled Nachname: {FORM_NACHNAME}")
|
|
||||||
form_filled = True
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning(f"[GEWOBAG] Could not fill Nachname: {e}")
|
|
||||||
|
|
||||||
# E-Mail
|
|
||||||
try:
|
|
||||||
email_field = await iframe_page.query_selector('#email')
|
|
||||||
if email_field:
|
|
||||||
await email_field.fill(FORM_EMAIL)
|
|
||||||
logger.info(f"[GEWOBAG] Filled E-Mail: {FORM_EMAIL}")
|
|
||||||
form_filled = True
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning(f"[GEWOBAG] Could not fill E-Mail: {e}")
|
|
||||||
|
|
||||||
# Telefonnummer - Gewobag uses #phone-number
|
|
||||||
try:
|
|
||||||
tel_field = await iframe_page.query_selector('#phone-number, input[id*="telefonnummer"], input[id*="phone"]')
|
|
||||||
if tel_field:
|
|
||||||
await tel_field.fill(FORM_PHONE)
|
|
||||||
logger.info(f"[GEWOBAG] Filled Telefon: {FORM_PHONE}")
|
|
||||||
form_filled = True
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning(f"[GEWOBAG] Could not fill Telefon: {e}")
|
|
||||||
|
|
||||||
# Anzahl einziehende Personen - Gewobag uses formly_*_gesamtzahl
|
|
||||||
try:
|
|
||||||
personen_field = await iframe_page.query_selector('input[id*="gesamtzahl"], input[id*="numberPersonsTotal"]')
|
|
||||||
if personen_field:
|
|
||||||
await personen_field.fill(FORM_PERSONS)
|
|
||||||
logger.info(f"[GEWOBAG] Filled Anzahl Personen: {FORM_PERSONS}")
|
|
||||||
form_filled = True
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning(f"[GEWOBAG] Could not fill Anzahl Personen: {e}")
|
|
||||||
|
|
||||||
await asyncio.sleep(1)
|
|
||||||
|
|
||||||
# Screenshot after filling
|
|
||||||
screenshot_path = DATA_DIR / f"gewobag_filled_{listing['id']}.png"
|
|
||||||
await iframe_page.screenshot(path=str(screenshot_path), full_page=True)
|
|
||||||
logger.info(f"[GEWOBAG] Saved filled form screenshot")
|
|
||||||
|
|
||||||
# Try to submit
|
|
||||||
if form_filled:
|
|
||||||
try:
|
|
||||||
submit_selectors = [
|
|
||||||
'button[type="submit"]',
|
|
||||||
'button:has-text("Absenden")',
|
|
||||||
'button:has-text("Senden")',
|
|
||||||
'button:has-text("Anfrage")',
|
|
||||||
'.btn-primary',
|
|
||||||
]
|
|
||||||
|
|
||||||
submit_btn = None
|
|
||||||
for selector in submit_selectors:
|
|
||||||
submit_btn = await iframe_page.query_selector(selector)
|
|
||||||
if submit_btn and await submit_btn.is_visible():
|
|
||||||
logger.info(f"[GEWOBAG] Found submit button: {selector}")
|
|
||||||
break
|
|
||||||
submit_btn = None
|
|
||||||
|
|
||||||
if submit_btn:
|
|
||||||
await submit_btn.click()
|
|
||||||
logger.info("[GEWOBAG] Clicked submit button")
|
|
||||||
await asyncio.sleep(3)
|
|
||||||
|
|
||||||
# Screenshot after submission
|
|
||||||
screenshot_path = DATA_DIR / f"gewobag_submitted_{listing['id']}.png"
|
|
||||||
await iframe_page.screenshot(path=str(screenshot_path), full_page=True)
|
|
||||||
logger.info(f"[GEWOBAG] Saved submission screenshot")
|
|
||||||
|
|
||||||
result["success"] = True
|
|
||||||
result["message"] = "Application submitted via Wohnungshelden"
|
|
||||||
else:
|
else:
|
||||||
result["success"] = False
|
result["message"] = "No application button found"
|
||||||
result["message"] = "Form filled but submit button not found"
|
logger.warning("[GEWOBAG] Could not find application button")
|
||||||
logger.warning("[GEWOBAG] Submit button not found")
|
|
||||||
except Exception as e:
|
|
||||||
result["success"] = False
|
|
||||||
result["message"] = f"Submit error: {str(e)}"
|
|
||||||
logger.warning(f"[GEWOBAG] Submit error: {e}")
|
|
||||||
else:
|
|
||||||
result["success"] = False
|
|
||||||
result["message"] = "No form fields found in Wohnungshelden"
|
|
||||||
logger.warning("[GEWOBAG] Could not find form fields")
|
|
||||||
finally:
|
|
||||||
await iframe_page.close()
|
|
||||||
else:
|
|
||||||
result["success"] = False
|
|
||||||
result["message"] = "No Wohnungshelden iframe found"
|
|
||||||
logger.warning("[GEWOBAG] No Wohnungshelden iframe found")
|
|
||||||
screenshot_path = DATA_DIR / f"gewobag_nobtn_{listing['id']}.png"
|
screenshot_path = DATA_DIR / f"gewobag_nobtn_{listing['id']}.png"
|
||||||
await page.screenshot(path=str(screenshot_path))
|
await page.screenshot(path=str(screenshot_path))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
result["success"] = False
|
|
||||||
result["message"] = f"Error: {str(e)}"
|
result["message"] = f"Error: {str(e)}"
|
||||||
logger.error(f"[GEWOBAG] Exception: {str(e)}")
|
logger.error(f"[GEWOBAG] Exception: {str(e)}")
|
||||||
finally:
|
finally:
|
||||||
|
|
@ -1052,11 +903,9 @@ class ApplicationHandler:
|
||||||
|
|
||||||
async def _apply_stadtundland(self, listing: dict, result: dict) -> dict:
|
async def _apply_stadtundland(self, listing: dict, result: dict) -> dict:
|
||||||
"""
|
"""
|
||||||
Stadt und Land has an embedded contact form directly on their listing page.
|
Stadt und Land uses Wohnungshelden (app.wohnungshelden.de) for their application system.
|
||||||
No iframe - the form fields are directly accessible.
|
The application form is loaded in an iframe from a different domain.
|
||||||
Fields: name, surname, street, houseNo, postalCode, city, phone, email
|
We need to navigate directly to the iframe URL.
|
||||||
Checkboxes: privacy, provision
|
|
||||||
Submit: "Eingaben prüfen"
|
|
||||||
"""
|
"""
|
||||||
page = await self.context.new_page()
|
page = await self.context.new_page()
|
||||||
try:
|
try:
|
||||||
|
|
@ -1074,196 +923,180 @@ class ApplicationHandler:
|
||||||
await asyncio.sleep(1)
|
await asyncio.sleep(1)
|
||||||
except: pass
|
except: pass
|
||||||
|
|
||||||
# Scroll down to the contact form
|
# Look for "Kontakt" or "Anfragen" button to open the form
|
||||||
await page.evaluate("window.scrollBy(0, 500)")
|
logger.info("[STADTUNDLAND] Looking for contact button...")
|
||||||
await asyncio.sleep(0.5)
|
apply_btn = await page.query_selector('a:has-text("Kontakt"), button:has-text("Kontakt"), a:has-text("Anfragen"), button:has-text("Anfragen"), a:has-text("kontaktieren"), button:has-text("kontaktieren")')
|
||||||
|
if apply_btn and await apply_btn.is_visible():
|
||||||
|
logger.info("[STADTUNDLAND] Found contact button, clicking...")
|
||||||
|
await apply_btn.click()
|
||||||
|
await asyncio.sleep(3)
|
||||||
|
|
||||||
# Take initial screenshot
|
# Stadt und Land uses Wohnungshelden iframe for the application form
|
||||||
screenshot_path = DATA_DIR / f"stadtundland_page_{listing['id']}.png"
|
iframe_element = await page.query_selector('iframe[src*="wohnungshelden.de"]')
|
||||||
await page.screenshot(path=str(screenshot_path), full_page=True)
|
if iframe_element:
|
||||||
logger.info(f"[STADTUNDLAND] Saved page screenshot to {screenshot_path}")
|
iframe_url = await iframe_element.get_attribute('src')
|
||||||
|
logger.info(f"[STADTUNDLAND] Found Wohnungshelden iframe: {iframe_url}")
|
||||||
|
|
||||||
# Fill out the embedded form directly
|
# Navigate to the iframe URL directly in a new page for full access
|
||||||
|
iframe_page = await self.context.new_page()
|
||||||
|
try:
|
||||||
|
await iframe_page.goto(iframe_url, wait_until="networkidle")
|
||||||
|
await asyncio.sleep(2)
|
||||||
|
logger.info("[STADTUNDLAND] Loaded Wohnungshelden application page")
|
||||||
|
|
||||||
|
# Take screenshot of the Wohnungshelden form
|
||||||
|
screenshot_path = DATA_DIR / f"stadtundland_wohnungshelden_{listing['id']}.png"
|
||||||
|
await iframe_page.screenshot(path=str(screenshot_path), full_page=True)
|
||||||
|
logger.info(f"[STADTUNDLAND] Saved Wohnungshelden screenshot to {screenshot_path}")
|
||||||
|
|
||||||
|
# Save HTML for debugging
|
||||||
|
html_content = await iframe_page.content()
|
||||||
|
html_path = DATA_DIR / f"stadtundland_wohnungshelden_{listing['id']}.html"
|
||||||
|
with open(html_path, 'w', encoding='utf-8') as f:
|
||||||
|
f.write(html_content)
|
||||||
|
logger.info(f"[STADTUNDLAND] Saved HTML to {html_path}")
|
||||||
|
|
||||||
|
# Fill out Wohnungshelden form (same as Degewo)
|
||||||
form_filled = False
|
form_filled = False
|
||||||
|
|
||||||
# Vorname (name field)
|
# Anrede (Salutation) - ng-select dropdown
|
||||||
try:
|
try:
|
||||||
vorname_field = await page.query_selector('input[name="name"]')
|
salutation_dropdown = await iframe_page.query_selector('#salutation-dropdown, ng-select[id*="salutation"]')
|
||||||
if vorname_field and await vorname_field.is_visible():
|
if salutation_dropdown:
|
||||||
|
await salutation_dropdown.click()
|
||||||
|
await asyncio.sleep(0.5)
|
||||||
|
anrede_option = await iframe_page.query_selector(f'.ng-option:has-text("{FORM_ANREDE}")')
|
||||||
|
if anrede_option:
|
||||||
|
await anrede_option.click()
|
||||||
|
logger.info(f"[STADTUNDLAND] Selected Anrede: {FORM_ANREDE}")
|
||||||
|
form_filled = True
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"[STADTUNDLAND] Could not set Anrede: {e}")
|
||||||
|
|
||||||
|
# Vorname (First name)
|
||||||
|
try:
|
||||||
|
vorname_field = await iframe_page.query_selector('#firstName')
|
||||||
|
if vorname_field:
|
||||||
await vorname_field.fill(FORM_VORNAME)
|
await vorname_field.fill(FORM_VORNAME)
|
||||||
logger.info(f"[STADTUNDLAND] Filled Vorname: {FORM_VORNAME}")
|
logger.info(f"[STADTUNDLAND] Filled Vorname: {FORM_VORNAME}")
|
||||||
form_filled = True
|
form_filled = True
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"[STADTUNDLAND] Could not fill Vorname: {e}")
|
logger.warning(f"[STADTUNDLAND] Could not fill Vorname: {e}")
|
||||||
|
|
||||||
# Nachname (surname field)
|
# Nachname (Last name)
|
||||||
try:
|
try:
|
||||||
nachname_field = await page.query_selector('input[name="surname"]')
|
nachname_field = await iframe_page.query_selector('#lastName')
|
||||||
if nachname_field and await nachname_field.is_visible():
|
if nachname_field:
|
||||||
await nachname_field.fill(FORM_NACHNAME)
|
await nachname_field.fill(FORM_NACHNAME)
|
||||||
logger.info(f"[STADTUNDLAND] Filled Nachname: {FORM_NACHNAME}")
|
logger.info(f"[STADTUNDLAND] Filled Nachname: {FORM_NACHNAME}")
|
||||||
form_filled = True
|
form_filled = True
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"[STADTUNDLAND] Could not fill Nachname: {e}")
|
logger.warning(f"[STADTUNDLAND] Could not fill Nachname: {e}")
|
||||||
|
|
||||||
# Straße (street field)
|
# E-Mail
|
||||||
try:
|
try:
|
||||||
street_field = await page.query_selector('input[name="street"]')
|
email_field = await iframe_page.query_selector('#email')
|
||||||
if street_field and await street_field.is_visible():
|
if email_field:
|
||||||
await street_field.fill(FORM_STRASSE)
|
|
||||||
logger.info(f"[STADTUNDLAND] Filled Straße: {FORM_STRASSE}")
|
|
||||||
form_filled = True
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning(f"[STADTUNDLAND] Could not fill Straße: {e}")
|
|
||||||
|
|
||||||
# Hausnummer (houseNo field)
|
|
||||||
try:
|
|
||||||
house_field = await page.query_selector('input[name="houseNo"]')
|
|
||||||
if house_field and await house_field.is_visible():
|
|
||||||
await house_field.fill(FORM_HAUSNUMMER)
|
|
||||||
logger.info(f"[STADTUNDLAND] Filled Hausnummer: {FORM_HAUSNUMMER}")
|
|
||||||
form_filled = True
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning(f"[STADTUNDLAND] Could not fill Hausnummer: {e}")
|
|
||||||
|
|
||||||
# PLZ (postalCode field)
|
|
||||||
try:
|
|
||||||
plz_field = await page.query_selector('input[name="postalCode"]')
|
|
||||||
if plz_field and await plz_field.is_visible():
|
|
||||||
await plz_field.fill(FORM_PLZ)
|
|
||||||
logger.info(f"[STADTUNDLAND] Filled PLZ: {FORM_PLZ}")
|
|
||||||
form_filled = True
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning(f"[STADTUNDLAND] Could not fill PLZ: {e}")
|
|
||||||
|
|
||||||
# Ort (city field)
|
|
||||||
try:
|
|
||||||
city_field = await page.query_selector('input[name="city"]')
|
|
||||||
if city_field and await city_field.is_visible():
|
|
||||||
await city_field.fill(FORM_ORT)
|
|
||||||
logger.info(f"[STADTUNDLAND] Filled Ort: {FORM_ORT}")
|
|
||||||
form_filled = True
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning(f"[STADTUNDLAND] Could not fill Ort: {e}")
|
|
||||||
|
|
||||||
# Telefon (phone field)
|
|
||||||
try:
|
|
||||||
phone_field = await page.query_selector('input[name="phone"]')
|
|
||||||
if phone_field and await phone_field.is_visible():
|
|
||||||
await phone_field.fill(FORM_PHONE)
|
|
||||||
logger.info(f"[STADTUNDLAND] Filled Telefon: {FORM_PHONE}")
|
|
||||||
form_filled = True
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning(f"[STADTUNDLAND] Could not fill Telefon: {e}")
|
|
||||||
|
|
||||||
# E-Mail (email field)
|
|
||||||
try:
|
|
||||||
email_field = await page.query_selector('input[name="email"]')
|
|
||||||
if email_field and await email_field.is_visible():
|
|
||||||
await email_field.fill(FORM_EMAIL)
|
await email_field.fill(FORM_EMAIL)
|
||||||
logger.info(f"[STADTUNDLAND] Filled E-Mail: {FORM_EMAIL}")
|
logger.info(f"[STADTUNDLAND] Filled E-Mail: {FORM_EMAIL}")
|
||||||
form_filled = True
|
form_filled = True
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"[STADTUNDLAND] Could not fill E-Mail: {e}")
|
logger.warning(f"[STADTUNDLAND] Could not fill E-Mail: {e}")
|
||||||
|
|
||||||
# Click privacy checkbox
|
# Telefonnummer
|
||||||
try:
|
try:
|
||||||
privacy_checkbox = await page.query_selector('input[name="privacy"]')
|
tel_field = await iframe_page.query_selector('input[id*="telefonnummer"]')
|
||||||
if privacy_checkbox and await privacy_checkbox.is_visible():
|
if tel_field:
|
||||||
if not await privacy_checkbox.is_checked():
|
await tel_field.fill(FORM_PHONE)
|
||||||
await privacy_checkbox.click()
|
logger.info(f"[STADTUNDLAND] Filled Telefon: {FORM_PHONE}")
|
||||||
logger.info("[STADTUNDLAND] Clicked privacy checkbox")
|
form_filled = True
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"[STADTUNDLAND] Could not click privacy checkbox: {e}")
|
logger.warning(f"[STADTUNDLAND] Could not fill Telefon: {e}")
|
||||||
|
|
||||||
# Click provision checkbox (optional)
|
# Anzahl einziehende Personen
|
||||||
try:
|
try:
|
||||||
provision_checkbox = await page.query_selector('input[name="provision"]')
|
personen_field = await iframe_page.query_selector('input[id*="numberPersonsTotal"]')
|
||||||
if provision_checkbox and await provision_checkbox.is_visible():
|
if personen_field:
|
||||||
if not await provision_checkbox.is_checked():
|
await personen_field.fill(FORM_PERSONS)
|
||||||
await provision_checkbox.click()
|
logger.info(f"[STADTUNDLAND] Filled Anzahl Personen: {FORM_PERSONS}")
|
||||||
logger.info("[STADTUNDLAND] Clicked provision checkbox")
|
form_filled = True
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"[STADTUNDLAND] Could not click provision checkbox: {e}")
|
logger.warning(f"[STADTUNDLAND] Could not fill Anzahl Personen: {e}")
|
||||||
|
|
||||||
|
# "Für sich selbst" dropdown
|
||||||
|
try:
|
||||||
|
selbst_dropdown = await iframe_page.query_selector('ng-select[id*="fuer_wen"]')
|
||||||
|
if selbst_dropdown:
|
||||||
|
await selbst_dropdown.click()
|
||||||
|
await asyncio.sleep(0.5)
|
||||||
|
selbst_option = await iframe_page.query_selector('.ng-option:has-text("Für mich selbst"), .ng-option:has-text("selbst")')
|
||||||
|
if selbst_option:
|
||||||
|
await selbst_option.click()
|
||||||
|
logger.info("[STADTUNDLAND] Selected: Für mich selbst")
|
||||||
|
form_filled = True
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"[STADTUNDLAND] Could not set 'Für sich selbst': {e}")
|
||||||
|
|
||||||
await asyncio.sleep(1)
|
await asyncio.sleep(1)
|
||||||
|
|
||||||
# Take screenshot after filling form
|
# Take screenshot after filling form
|
||||||
screenshot_path = DATA_DIR / f"stadtundland_filled_{listing['id']}.png"
|
screenshot_path = DATA_DIR / f"stadtundland_form_filled_{listing['id']}.png"
|
||||||
await page.screenshot(path=str(screenshot_path), full_page=True)
|
await iframe_page.screenshot(path=str(screenshot_path), full_page=True)
|
||||||
logger.info(f"[STADTUNDLAND] Saved filled form screenshot to {screenshot_path}")
|
logger.info(f"[STADTUNDLAND] Saved filled form screenshot to {screenshot_path}")
|
||||||
|
|
||||||
# Submit form
|
# Try to submit
|
||||||
if form_filled:
|
if form_filled:
|
||||||
try:
|
try:
|
||||||
# Stadt und Land uses "Eingaben prüfen" button
|
submit_selectors = [
|
||||||
# Step 1: Click "Eingaben prüfen" button
|
'button[type="submit"]',
|
||||||
pruefen_btn = await page.query_selector('button:has-text("Eingaben prüfen")')
|
'input[type="submit"]',
|
||||||
if pruefen_btn and await pruefen_btn.is_visible():
|
|
||||||
await pruefen_btn.click()
|
|
||||||
logger.info("[STADTUNDLAND] Clicked 'Eingaben prüfen' button")
|
|
||||||
await asyncio.sleep(2)
|
|
||||||
await page.wait_for_load_state("networkidle")
|
|
||||||
|
|
||||||
# Take screenshot after validation
|
|
||||||
screenshot_path = DATA_DIR / f"stadtundland_validated_{listing['id']}.png"
|
|
||||||
await page.screenshot(path=str(screenshot_path), full_page=True)
|
|
||||||
logger.info(f"[STADTUNDLAND] Saved validation screenshot")
|
|
||||||
|
|
||||||
# Step 2: Click the final submit button
|
|
||||||
final_submit_selectors = [
|
|
||||||
'button:has-text("Absenden")',
|
'button:has-text("Absenden")',
|
||||||
'button:has-text("Senden")',
|
'button:has-text("Senden")',
|
||||||
'button:has-text("Anfrage senden")',
|
'button:has-text("Anfrage")',
|
||||||
'button:has-text("Bestätigen")',
|
'.btn-primary',
|
||||||
'button[type="submit"]',
|
|
||||||
]
|
]
|
||||||
|
|
||||||
final_btn = None
|
submit_btn = None
|
||||||
for selector in final_submit_selectors:
|
for selector in submit_selectors:
|
||||||
final_btn = await page.query_selector(selector)
|
submit_btn = await iframe_page.query_selector(selector)
|
||||||
if final_btn and await final_btn.is_visible():
|
if submit_btn and await submit_btn.is_visible():
|
||||||
logger.info(f"[STADTUNDLAND] Found final submit button: {selector}")
|
logger.info(f"[STADTUNDLAND] Found submit button with selector: {selector}")
|
||||||
break
|
break
|
||||||
final_btn = None
|
submit_btn = None
|
||||||
|
|
||||||
if final_btn:
|
if submit_btn:
|
||||||
await final_btn.click()
|
await submit_btn.click()
|
||||||
logger.info("[STADTUNDLAND] Clicked final submit button")
|
logger.info("[STADTUNDLAND] Clicked submit button")
|
||||||
await asyncio.sleep(3)
|
await asyncio.sleep(3)
|
||||||
await page.wait_for_load_state("networkidle")
|
|
||||||
|
|
||||||
# Take screenshot after final submission
|
# Take screenshot after submission
|
||||||
screenshot_path = DATA_DIR / f"stadtundland_submitted_{listing['id']}.png"
|
screenshot_path = DATA_DIR / f"stadtundland_submitted_{listing['id']}.png"
|
||||||
await page.screenshot(path=str(screenshot_path), full_page=True)
|
await iframe_page.screenshot(path=str(screenshot_path), full_page=True)
|
||||||
logger.info(f"[STADTUNDLAND] Saved submission screenshot")
|
logger.info(f"[STADTUNDLAND] Saved submission screenshot to {screenshot_path}")
|
||||||
|
|
||||||
# Check for confirmation message
|
|
||||||
content = await page.content()
|
|
||||||
if "erfolgreich" in content.lower() or "gesendet" in content.lower() or "danke" in content.lower() or "bestätigung" in content.lower():
|
|
||||||
result["success"] = True
|
result["success"] = True
|
||||||
result["message"] = "Application submitted successfully"
|
result["message"] = "Application submitted via Wohnungshelden"
|
||||||
logger.info("[STADTUNDLAND] Success! Confirmation message detected")
|
|
||||||
else:
|
|
||||||
result["success"] = True
|
|
||||||
result["message"] = "Form submitted"
|
|
||||||
logger.info("[STADTUNDLAND] Form submitted")
|
|
||||||
else:
|
else:
|
||||||
result["success"] = False
|
result["success"] = False
|
||||||
result["message"] = "Validated but final submit button not found"
|
result["message"] = "Form filled, submit button not found"
|
||||||
logger.warning("[STADTUNDLAND] Final submit button not found")
|
logger.warning("[STADTUNDLAND] Submit button not found")
|
||||||
else:
|
|
||||||
result["success"] = False
|
|
||||||
result["message"] = "Form filled but 'Eingaben prüfen' button not found"
|
|
||||||
logger.warning("[STADTUNDLAND] 'Eingaben prüfen' button not found")
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
result["success"] = False
|
result["success"] = False
|
||||||
result["message"] = f"Submit error: {str(e)}"
|
result["message"] = f"Form filled, submit error: {str(e)}"
|
||||||
logger.warning(f"[STADTUNDLAND] Submit error: {e}")
|
logger.warning(f"[STADTUNDLAND] Submit error: {e}")
|
||||||
else:
|
else:
|
||||||
result["success"] = False
|
result["success"] = False
|
||||||
result["message"] = "No form fields found on page"
|
result["message"] = "No form fields found in Wohnungshelden"
|
||||||
logger.warning("[STADTUNDLAND] Could not find form fields")
|
logger.warning("[STADTUNDLAND] Could not find form fields in Wohnungshelden")
|
||||||
|
finally:
|
||||||
|
await iframe_page.close()
|
||||||
|
else:
|
||||||
|
# No Wohnungshelden iframe found
|
||||||
|
result["success"] = False
|
||||||
|
result["message"] = "No Wohnungshelden iframe found"
|
||||||
|
logger.warning("[STADTUNDLAND] No Wohnungshelden iframe found")
|
||||||
|
screenshot_path = DATA_DIR / f"stadtundland_nobtn_{listing['id']}.png"
|
||||||
|
await page.screenshot(path=str(screenshot_path), full_page=True)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
result["success"] = False
|
result["success"] = False
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue