working app
This commit is contained in:
parent
8e69e30387
commit
3057cda8d3
12 changed files with 708 additions and 232 deletions
|
|
@ -1,6 +1,20 @@
|
|||
from .base_handler import BaseHandler
|
||||
|
||||
import logging
|
||||
import asyncio
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
# Load environment variables for form fields and data dir
|
||||
FORM_VORNAME = os.environ.get("FORM_VORNAME", "")
|
||||
FORM_NACHNAME = os.environ.get("FORM_NACHNAME", "")
|
||||
FORM_STRASSE = os.environ.get("FORM_STRASSE", "")
|
||||
FORM_HAUSNUMMER = os.environ.get("FORM_HAUSNUMMER", "")
|
||||
FORM_PLZ = os.environ.get("FORM_PLZ", "")
|
||||
FORM_ORT = os.environ.get("FORM_ORT", "")
|
||||
FORM_PHONE = os.environ.get("FORM_PHONE", "")
|
||||
FORM_EMAIL = os.environ.get("FORM_EMAIL", "")
|
||||
DATA_DIR = Path(os.environ.get("DATA_DIR", "data"))
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -11,77 +25,178 @@ class StadtUndLandHandler(BaseHandler):
|
|||
async def apply(self, listing: dict, result: dict) -> dict:
|
||||
page = await self.context.new_page()
|
||||
try:
|
||||
logger.info(f"[STADT UND LAND] Open: {listing['link']}")
|
||||
logger.info(f"[STADTUNDLAND] Opening page: {listing['link']}")
|
||||
await page.goto(listing["link"], wait_until="networkidle")
|
||||
logger.info("[STADTUNDLAND] Page loaded")
|
||||
await asyncio.sleep(2)
|
||||
|
||||
# Always handle cookies and consent before anything else
|
||||
await self.handle_cookies(page)
|
||||
await self.handle_consent(page)
|
||||
|
||||
# Save HTML after modal handling for debugging
|
||||
# Dismiss cookie banner
|
||||
try:
|
||||
html_content = await page.content()
|
||||
with open("data/stadtundland_debug.html", "w", encoding="utf-8") as f:
|
||||
f.write(html_content)
|
||||
cookie_btn = await page.query_selector('button:has-text("Akzeptieren"), button:has-text("Alle akzeptieren")')
|
||||
if cookie_btn and await cookie_btn.is_visible():
|
||||
await cookie_btn.click()
|
||||
logger.info("[STADTUNDLAND] Dismissed cookie banner")
|
||||
await asyncio.sleep(1)
|
||||
except Exception as e:
|
||||
logger.debug(f"[STADT UND LAND] Debug HTML not saved: {e}")
|
||||
logger.debug(f"[STADTUNDLAND] Cookie banner dismiss failed: {e}")
|
||||
|
||||
# 404/permanent fail detection
|
||||
error_texts = [
|
||||
"Hier ist etwas schief gelaufen",
|
||||
"Leider können wir Ihnen zur Zeit keine Details zu diesem Inserat anzeigen"
|
||||
]
|
||||
page_text = await page.text_content('body')
|
||||
if page_text:
|
||||
for err in error_texts:
|
||||
if err in page_text:
|
||||
logger.warning(f"[STADT UND LAND] Permanent fail: {err}")
|
||||
result["permanent_fail"] = True
|
||||
result["message"] = "Listing is no longer available (404 detected on STADT UND LAND)."
|
||||
await page.close()
|
||||
return result
|
||||
# Scroll to form
|
||||
await page.evaluate("window.scrollBy(0, 500)")
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# Look for application button (robust selectors)
|
||||
logger.info("[STADT UND LAND] Searching for application button...")
|
||||
selectors = [
|
||||
'a[href*="bewerben"]',
|
||||
'button:has-text("Bewerben")',
|
||||
'a:has-text("Bewerben")',
|
||||
'button.btn',
|
||||
'a.Button_button__JnZ4E',
|
||||
'button.Button_button__JnZ4E',
|
||||
]
|
||||
# Fill out the embedded form directly
|
||||
form_filled = False
|
||||
try:
|
||||
# Vorname
|
||||
vorname_field = await page.query_selector('input[name="name"]')
|
||||
if vorname_field and await vorname_field.is_visible():
|
||||
await vorname_field.fill(FORM_VORNAME)
|
||||
logger.info(f"[STADTUNDLAND] Filled Vorname: {FORM_VORNAME}")
|
||||
form_filled = True
|
||||
# Nachname
|
||||
nachname_field = await page.query_selector('input[name="surname"]')
|
||||
if nachname_field and await nachname_field.is_visible():
|
||||
await nachname_field.fill(FORM_NACHNAME)
|
||||
logger.info(f"[STADTUNDLAND] Filled Nachname: {FORM_NACHNAME}")
|
||||
form_filled = True
|
||||
# Straße
|
||||
street_field = await page.query_selector('input[name="street"]')
|
||||
if street_field and await street_field.is_visible():
|
||||
await street_field.fill(FORM_STRASSE)
|
||||
logger.info(f"[STADTUNDLAND] Filled Straße: {FORM_STRASSE}")
|
||||
form_filled = True
|
||||
# Hausnummer
|
||||
house_field = await page.query_selector('input[name="houseNo"]')
|
||||
if house_field and await house_field.is_visible():
|
||||
await house_field.fill(FORM_HAUSNUMMER)
|
||||
logger.info(f"[STADTUNDLAND] Filled Hausnummer: {FORM_HAUSNUMMER}")
|
||||
form_filled = True
|
||||
# PLZ
|
||||
plz_field = await page.query_selector('input[name="postalCode"]')
|
||||
if plz_field and await plz_field.is_visible():
|
||||
await plz_field.fill(FORM_PLZ)
|
||||
logger.info(f"[STADTUNDLAND] Filled PLZ: {FORM_PLZ}")
|
||||
form_filled = True
|
||||
# Ort
|
||||
city_field = await page.query_selector('input[name="city"]')
|
||||
if city_field and await city_field.is_visible():
|
||||
await city_field.fill(FORM_ORT)
|
||||
logger.info(f"[STADTUNDLAND] Filled Ort: {FORM_ORT}")
|
||||
form_filled = True
|
||||
# Telefon
|
||||
phone_field = await page.query_selector('input[name="phone"]')
|
||||
if phone_field and await phone_field.is_visible():
|
||||
await phone_field.fill(FORM_PHONE)
|
||||
logger.info(f"[STADTUNDLAND] Filled Telefon: {FORM_PHONE}")
|
||||
form_filled = True
|
||||
# E-Mail
|
||||
email_field = await page.query_selector('input[name="email"]')
|
||||
if email_field and await email_field.is_visible():
|
||||
await email_field.fill(FORM_EMAIL)
|
||||
logger.info(f"[STADTUNDLAND] Filled E-Mail: {FORM_EMAIL}")
|
||||
form_filled = True
|
||||
except Exception as e:
|
||||
logger.warning(f"[STADTUNDLAND] Error filling form fields: {e}")
|
||||
|
||||
apply_btn = None
|
||||
for sel in selectors:
|
||||
all_btns = await page.query_selector_all(sel)
|
||||
logger.debug(f"[STADT UND LAND] Selector '{sel}': {len(all_btns)} matches")
|
||||
for btn in all_btns:
|
||||
try:
|
||||
if await btn.is_visible():
|
||||
apply_btn = btn
|
||||
logger.info(f"[STADT UND LAND] Found visible application button: {sel}")
|
||||
break
|
||||
except Exception as e:
|
||||
logger.debug(f"[STADT UND LAND] Button visibility error: {e}")
|
||||
if apply_btn:
|
||||
break
|
||||
# Click privacy checkbox
|
||||
try:
|
||||
privacy_checkbox = await page.query_selector('input[name="privacy"]')
|
||||
if privacy_checkbox and await privacy_checkbox.is_visible():
|
||||
if not await privacy_checkbox.is_checked():
|
||||
await privacy_checkbox.click()
|
||||
logger.info("[STADTUNDLAND] Clicked privacy checkbox")
|
||||
except Exception as e:
|
||||
logger.warning(f"[STADTUNDLAND] Could not click privacy checkbox: {e}")
|
||||
|
||||
if apply_btn:
|
||||
await apply_btn.scroll_into_view_if_needed()
|
||||
await asyncio.sleep(0.5)
|
||||
await apply_btn.click()
|
||||
await asyncio.sleep(2)
|
||||
result["success"] = True
|
||||
result["message"] = "Application submitted successfully."
|
||||
# Click provision checkbox (optional)
|
||||
try:
|
||||
provision_checkbox = await page.query_selector('input[name="provision"]')
|
||||
if provision_checkbox and await provision_checkbox.is_visible():
|
||||
if not await provision_checkbox.is_checked():
|
||||
await provision_checkbox.click()
|
||||
logger.info("[STADTUNDLAND] Clicked provision checkbox")
|
||||
except Exception as e:
|
||||
logger.warning(f"[STADTUNDLAND] Could not click provision checkbox: {e}")
|
||||
|
||||
await asyncio.sleep(1)
|
||||
|
||||
# Screenshot after filling form
|
||||
screenshot_path2 = DATA_DIR / f"stadtundland_filled_{listing['id']}.png"
|
||||
await page.screenshot(path=str(screenshot_path2), full_page=True)
|
||||
logger.info(f"[STADTUNDLAND] Saved filled form screenshot to {screenshot_path2}")
|
||||
|
||||
# Submit form
|
||||
if form_filled:
|
||||
try:
|
||||
pruefen_btn = await page.query_selector('button:has-text("Eingaben prüfen")')
|
||||
if pruefen_btn and await pruefen_btn.is_visible():
|
||||
await pruefen_btn.click()
|
||||
logger.info("[STADTUNDLAND] Clicked 'Eingaben prüfen' button")
|
||||
await asyncio.sleep(2)
|
||||
await page.wait_for_load_state("networkidle")
|
||||
|
||||
# Screenshot after validation
|
||||
screenshot_path3 = DATA_DIR / f"stadtundland_validated_{listing['id']}.png"
|
||||
await page.screenshot(path=str(screenshot_path3), full_page=True)
|
||||
logger.info(f"[STADTUNDLAND] Saved validation screenshot to {screenshot_path3}")
|
||||
|
||||
# Final submit
|
||||
final_submit_selectors = [
|
||||
'button:has-text("Absenden")',
|
||||
'button:has-text("Senden")',
|
||||
'button:has-text("Anfrage senden")',
|
||||
'button:has-text("Bestätigen")',
|
||||
'button[type="submit"]',
|
||||
]
|
||||
final_btn = None
|
||||
for selector in final_submit_selectors:
|
||||
btn = await page.query_selector(selector)
|
||||
if btn and await btn.is_visible():
|
||||
final_btn = btn
|
||||
logger.info(f"[STADTUNDLAND] Found final submit button: {selector}")
|
||||
break
|
||||
if final_btn:
|
||||
await final_btn.click()
|
||||
logger.info("[STADTUNDLAND] Clicked final submit button")
|
||||
await asyncio.sleep(3)
|
||||
await page.wait_for_load_state("networkidle")
|
||||
|
||||
# Screenshot after submission
|
||||
screenshot_path4 = DATA_DIR / f"stadtundland_submitted_{listing['id']}.png"
|
||||
await page.screenshot(path=str(screenshot_path4), full_page=True)
|
||||
logger.info(f"[STADTUNDLAND] Saved submission screenshot to {screenshot_path4}")
|
||||
|
||||
# Check for confirmation
|
||||
content = await page.content()
|
||||
if any(word in content.lower() for word in ["erfolgreich", "gesendet", "danke", "bestätigung"]):
|
||||
result["success"] = True
|
||||
result["message"] = "Application submitted successfully"
|
||||
logger.info("[STADTUNDLAND] Success! Confirmation message detected")
|
||||
else:
|
||||
result["success"] = True
|
||||
result["message"] = "Form submitted"
|
||||
logger.info("[STADTUNDLAND] Form submitted")
|
||||
else:
|
||||
result["success"] = False
|
||||
result["message"] = "Validated but final submit button not found"
|
||||
logger.warning("[STADTUNDLAND] Final submit button not found")
|
||||
else:
|
||||
result["success"] = False
|
||||
result["message"] = "Form filled but 'Eingaben prüfen' button not found"
|
||||
logger.warning("[STADTUNDLAND] 'Eingaben prüfen' button not found")
|
||||
except Exception as e:
|
||||
result["success"] = False
|
||||
result["message"] = f"Submit error: {str(e)}"
|
||||
logger.warning(f"[STADTUNDLAND] Submit error: {e}")
|
||||
else:
|
||||
logger.warning("[STADT UND LAND] No application button found.")
|
||||
result["message"] = "No application button found."
|
||||
result["success"] = False
|
||||
result["message"] = "No form fields found on page"
|
||||
logger.warning("[STADTUNDLAND] Could not find form fields")
|
||||
|
||||
except Exception as e:
|
||||
result["message"] = f"Error during application: {e}"
|
||||
logger.error(f"[STADT UND LAND] Application error: {e}")
|
||||
result["success"] = False
|
||||
result["message"] = f"Error: {str(e)}"
|
||||
logger.error(f"[STADTUNDLAND] Exception: {str(e)}")
|
||||
finally:
|
||||
await page.close()
|
||||
|
||||
return result
|
||||
Loading…
Add table
Add a link
Reference in a new issue