working app
This commit is contained in:
parent
8e69e30387
commit
3057cda8d3
12 changed files with 708 additions and 232 deletions
|
|
@ -1,6 +1,9 @@
|
|||
from .base_handler import BaseHandler
|
||||
|
||||
from handlers.base_handler import BaseHandler
|
||||
import logging
|
||||
import asyncio
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -9,13 +12,16 @@ class DegewoHandler(BaseHandler):
|
|||
self.context = browser_context
|
||||
|
||||
async def apply(self, listing: dict, result: dict) -> dict:
|
||||
DATA_DIR = Path("data/degewo")
|
||||
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
page = await self.context.new_page()
|
||||
try:
|
||||
logger.info(f"[DEGEWO] Open: {listing['link']}")
|
||||
logger.info(f"[DEGEWO] Opening page: {listing['link']}")
|
||||
response = await page.goto(listing["link"], wait_until="networkidle")
|
||||
logger.info("[DEGEWO] Page loaded")
|
||||
await asyncio.sleep(2)
|
||||
|
||||
# Detect 404 by status or page title
|
||||
# 404 detection
|
||||
status = response.status if response else None
|
||||
page_title = await page.title()
|
||||
if status == 404 or (page_title and "404" in page_title):
|
||||
|
|
@ -23,63 +29,67 @@ class DegewoHandler(BaseHandler):
|
|||
result["success"] = False
|
||||
result["message"] = "Listing is no longer available (404). Application impossible. Will not retry."
|
||||
result["permanent_fail"] = True
|
||||
await page.close()
|
||||
return result
|
||||
|
||||
# Always handle cookies and consent before anything else
|
||||
await self.handle_cookies(page)
|
||||
await self.handle_consent(page)
|
||||
# Check for 'INSERAT DEAKTIVIERT' (deactivated listing)
|
||||
page_content = await page.content()
|
||||
if "INSERAT DEAKTIVIERT" in page_content or "Inserat deaktiviert" in page_content:
|
||||
logger.warning("[DEGEWO] Listing is deactivated (INSERAT DEAKTIVIERT detected), treating as 404")
|
||||
result["success"] = False
|
||||
result["message"] = "Listing deactivated (404)"
|
||||
result["deactivated"] = True # Mark for removal from retries
|
||||
await page.close()
|
||||
return result
|
||||
|
||||
# Save HTML after modal handling for debugging
|
||||
# Dismiss cookie banner
|
||||
try:
|
||||
html_content = await page.content()
|
||||
with open("data/degewo_debug.html", "w", encoding="utf-8") as f:
|
||||
f.write(html_content)
|
||||
cookie_btn = await page.query_selector('button:has-text("Alle akzeptieren"), #CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll')
|
||||
if cookie_btn and await cookie_btn.is_visible():
|
||||
await cookie_btn.click()
|
||||
logger.info("[DEGEWO] Dismissed cookie banner")
|
||||
await asyncio.sleep(1)
|
||||
except Exception as e:
|
||||
logger.debug(f"[DEGEWO] Debug HTML not saved: {e}")
|
||||
logger.debug(f"[DEGEWO] Cookie banner dismiss failed: {e}")
|
||||
|
||||
logger.info("[DEGEWO] Searching for application button...")
|
||||
selectors = [
|
||||
'a.btn',
|
||||
'button.btn',
|
||||
'a:has-text("Bewerben")',
|
||||
'button:has-text("Bewerben")',
|
||||
'a:has-text("Anfrage")',
|
||||
'button:has-text("Anfrage")',
|
||||
'a:has-text("Kontakt")',
|
||||
'button:has-text("Kontakt")',
|
||||
]
|
||||
apply_btn = None
|
||||
for sel in selectors:
|
||||
all_btns = await page.query_selector_all(sel)
|
||||
logger.debug(f"[DEGEWO] Selector '{sel}': {len(all_btns)} matches")
|
||||
for btn in all_btns:
|
||||
try:
|
||||
if await btn.is_visible():
|
||||
btn_text = (await btn.inner_text()).lower()
|
||||
if any(x in btn_text for x in ["drucken", "merken", "zurück"]):
|
||||
continue
|
||||
apply_btn = btn
|
||||
logger.info(f"[DEGEWO] Found visible application button: {sel} [{btn_text}]")
|
||||
break
|
||||
except Exception as e:
|
||||
logger.debug(f"[DEGEWO] Button visibility error: {e}")
|
||||
if apply_btn:
|
||||
break
|
||||
|
||||
if apply_btn:
|
||||
await apply_btn.scroll_into_view_if_needed()
|
||||
await asyncio.sleep(0.5)
|
||||
logger.info("[DEGEWO] Looking for kontaktieren button...")
|
||||
apply_btn = await page.query_selector('a:has-text("kontaktieren"), button:has-text("kontaktieren"), a:has-text("Kontaktieren"), button:has-text("Kontaktieren")')
|
||||
if apply_btn and await apply_btn.is_visible():
|
||||
logger.info("[DEGEWO] Found kontaktieren button, clicking...")
|
||||
await apply_btn.click()
|
||||
await asyncio.sleep(2)
|
||||
result["success"] = True
|
||||
result["message"] = "Application submitted successfully."
|
||||
else:
|
||||
logger.warning("[DEGEWO] No application button found.")
|
||||
result["message"] = "No application button found."
|
||||
except Exception as e:
|
||||
result["message"] = f"Error during application: {e}"
|
||||
logger.error(f"[DEGEWO] Application error: {e}")
|
||||
finally:
|
||||
await page.close()
|
||||
await asyncio.sleep(3)
|
||||
|
||||
return result
|
||||
# Degewo uses Wohnungshelden iframe for the application form
|
||||
# Find the iframe and get its URL to navigate directly
|
||||
iframe_element = await page.query_selector('iframe[src*="wohnungshelden.de"]')
|
||||
if iframe_element:
|
||||
iframe_url = await iframe_element.get_attribute('src')
|
||||
logger.info(f"[DEGEWO] Found Wohnungshelden iframe: {iframe_url}")
|
||||
|
||||
# Navigate to the iframe URL directly in a new page for full access
|
||||
iframe_page = await self.context.new_page()
|
||||
try:
|
||||
await iframe_page.goto(iframe_url, wait_until="networkidle")
|
||||
await asyncio.sleep(2)
|
||||
logger.info("[DEGEWO] Loaded Wohnungshelden application page")
|
||||
# TODO: Implement form-filling and submission logic here
|
||||
finally:
|
||||
await iframe_page.close()
|
||||
else:
|
||||
# No iframe found - try the old approach (fallback for different page structure)
|
||||
logger.warning("[DEGEWO] Wohnungshelden iframe not found, trying direct form...")
|
||||
# TODO: Implement fallback logic here
|
||||
else:
|
||||
result["message"] = "No kontaktieren button found"
|
||||
logger.warning("[DEGEWO] Could not find kontaktieren button")
|
||||
screenshot_path = DATA_DIR / f"degewo_nobtn_{listing['id']}.png"
|
||||
await page.screenshot(path=str(screenshot_path), full_page=True)
|
||||
|
||||
await page.close()
|
||||
return result
|
||||
except Exception as e:
|
||||
result["success"] = False
|
||||
result["message"] = f"Error: {str(e)}"
|
||||
logger.error(f"[DEGEWO] Exception: {str(e)}")
|
||||
await page.close()
|
||||
return result
|
||||
|
|
@ -11,10 +11,22 @@ class GesobauHandler(BaseHandler):
|
|||
async def apply(self, listing: dict, result: dict) -> dict:
|
||||
page = await self.context.new_page()
|
||||
try:
|
||||
logger.info(f"[GESOBAU] Open: {listing['link']}")
|
||||
await page.goto(listing["link"], wait_until="networkidle")
|
||||
logger.info(f"[GESOBAU] Opening page: {listing['link']}")
|
||||
response = await page.goto(listing["link"], wait_until="networkidle")
|
||||
logger.info("[GESOBAU] Page loaded")
|
||||
await asyncio.sleep(2)
|
||||
|
||||
# 404 detection
|
||||
status = response.status if response else None
|
||||
page_title = await page.title()
|
||||
if status == 404 or (page_title and "404" in page_title):
|
||||
logger.warning(f"[GESOBAU] Listing is down (404): {listing['link']}")
|
||||
result["success"] = False
|
||||
result["message"] = "Listing is no longer available (404). Application impossible. Will not retry."
|
||||
result["permanent_fail"] = True
|
||||
await page.close()
|
||||
return result
|
||||
|
||||
# Always handle cookies and consent before anything else
|
||||
await self.handle_cookies(page)
|
||||
await self.handle_consent(page)
|
||||
|
|
@ -63,8 +75,47 @@ class GesobauHandler(BaseHandler):
|
|||
await asyncio.sleep(0.5)
|
||||
await apply_btn.click()
|
||||
await asyncio.sleep(2)
|
||||
result["success"] = True
|
||||
result["message"] = "Application submitted successfully."
|
||||
# --- Post-click confirmation logic ---
|
||||
logger.info("[GESOBAU] Clicked application button, checking for confirmation...")
|
||||
# Save screenshot and HTML after click
|
||||
try:
|
||||
await page.screenshot(path="data/gesobau_after_apply.png")
|
||||
logger.info("[GESOBAU] Saved screenshot after application click.")
|
||||
except Exception as e:
|
||||
logger.warning(f"[GESOBAU] Could not save screenshot: {e}")
|
||||
try:
|
||||
html_after = await page.content()
|
||||
with open("data/gesobau_after_apply.html", "w", encoding="utf-8") as f:
|
||||
f.write(html_after)
|
||||
logger.info("[GESOBAU] Saved HTML after application click.")
|
||||
except Exception as e:
|
||||
logger.warning(f"[GESOBAU] Could not save HTML after apply: {e}")
|
||||
|
||||
# Look for confirmation message on the page
|
||||
confirmation_selectors = [
|
||||
'text="Vielen Dank"',
|
||||
'text="Ihre Anfrage wurde gesendet"',
|
||||
'text="Bestätigung"',
|
||||
'div:has-text("Vielen Dank")',
|
||||
'div:has-text("Ihre Anfrage wurde gesendet")',
|
||||
]
|
||||
confirmed = False
|
||||
for sel in confirmation_selectors:
|
||||
try:
|
||||
el = await page.query_selector(sel)
|
||||
if el and await el.is_visible():
|
||||
logger.info(f"[GESOBAU] Found confirmation element: {sel}")
|
||||
confirmed = True
|
||||
break
|
||||
except Exception as e:
|
||||
logger.debug(f"[GESOBAU] Error checking confirmation selector {sel}: {e}")
|
||||
if confirmed:
|
||||
result["success"] = True
|
||||
result["message"] = "Application submitted and confirmation detected."
|
||||
else:
|
||||
logger.warning("[GESOBAU] No confirmation message detected after application click.")
|
||||
result["success"] = False
|
||||
result["message"] = "Clicked application button, but no confirmation detected. Check screenshot and HTML."
|
||||
else:
|
||||
logger.warning("[GESOBAU] No application button found.")
|
||||
result["message"] = "No application button found."
|
||||
|
|
|
|||
|
|
@ -90,8 +90,47 @@ class GewobagHandler(BaseHandler):
|
|||
logger.info("[GEWOBAG] Clicking button...")
|
||||
await apply_btn.click()
|
||||
await asyncio.sleep(2)
|
||||
result["success"] = True
|
||||
result["message"] = "Application submitted successfully."
|
||||
# --- Post-click confirmation logic ---
|
||||
logger.info("[GEWOBAG] Clicked application button, checking for confirmation...")
|
||||
# Save screenshot and HTML after click
|
||||
try:
|
||||
await page.screenshot(path="data/gewobag_after_apply.png")
|
||||
logger.info("[GEWOBAG] Saved screenshot after application click.")
|
||||
except Exception as e:
|
||||
logger.warning(f"[GEWOBAG] Could not save screenshot: {e}")
|
||||
try:
|
||||
html_after = await page.content()
|
||||
with open("data/gewobag_after_apply.html", "w", encoding="utf-8") as f:
|
||||
f.write(html_after)
|
||||
logger.info("[GEWOBAG] Saved HTML after application click.")
|
||||
except Exception as e:
|
||||
logger.warning(f"[GEWOBAG] Could not save HTML after apply: {e}")
|
||||
|
||||
# Look for confirmation message on the page
|
||||
confirmation_selectors = [
|
||||
'text="Vielen Dank"',
|
||||
'text="Ihre Anfrage wurde gesendet"',
|
||||
'text="Bestätigung"',
|
||||
'div:has-text("Vielen Dank")',
|
||||
'div:has-text("Ihre Anfrage wurde gesendet")',
|
||||
]
|
||||
confirmed = False
|
||||
for sel in confirmation_selectors:
|
||||
try:
|
||||
el = await page.query_selector(sel)
|
||||
if el and await el.is_visible():
|
||||
logger.info(f"[GEWOBAG] Found confirmation element: {sel}")
|
||||
confirmed = True
|
||||
break
|
||||
except Exception as e:
|
||||
logger.debug(f"[GEWOBAG] Error checking confirmation selector {sel}: {e}")
|
||||
if confirmed:
|
||||
result["success"] = True
|
||||
result["message"] = "Application submitted and confirmation detected."
|
||||
else:
|
||||
logger.warning("[GEWOBAG] No confirmation message detected after application click.")
|
||||
result["success"] = False
|
||||
result["message"] = "Clicked application button, but no confirmation detected. Check screenshot and HTML."
|
||||
else:
|
||||
result["message"] = "No application button found."
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -9,10 +9,15 @@ class HowogeHandler(BaseHandler):
|
|||
self.context = browser_context
|
||||
|
||||
async def apply(self, listing: dict, result: dict) -> dict:
|
||||
import os
|
||||
from pathlib import Path
|
||||
DATA_DIR = Path("data/howoge")
|
||||
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
page = await self.context.new_page()
|
||||
try:
|
||||
logger.info(f"[HOWOGE] Open: {listing['link']}")
|
||||
logger.info(f"[HOWOGE] Opening page: {listing['link']}")
|
||||
response = await page.goto(listing["link"], wait_until="networkidle")
|
||||
logger.info("[HOWOGE] Page loaded")
|
||||
await asyncio.sleep(2)
|
||||
|
||||
# Detect 404 by status or page title
|
||||
|
|
@ -23,23 +28,35 @@ class HowogeHandler(BaseHandler):
|
|||
result["success"] = False
|
||||
result["message"] = "Listing is no longer available (404). Application impossible. Will not retry."
|
||||
result["permanent_fail"] = True
|
||||
await page.close()
|
||||
return result
|
||||
|
||||
# Always handle cookies and consent before anything else
|
||||
await self.handle_cookies(page)
|
||||
await self.handle_consent(page)
|
||||
|
||||
# Save HTML after modal handling for debugging
|
||||
# Handle cookies
|
||||
try:
|
||||
html_content = await page.content()
|
||||
with open("data/howoge_debug.html", "w", encoding="utf-8") as f:
|
||||
f.write(html_content)
|
||||
except Exception as e:
|
||||
logger.debug(f"[HOWOGE] Debug HTML not saved: {e}")
|
||||
cookie_btn = await page.query_selector('button:has-text("Akzeptieren"), button:has-text("Alle akzeptieren")')
|
||||
if cookie_btn and await cookie_btn.is_visible():
|
||||
await cookie_btn.click()
|
||||
logger.info("[HOWOGE] Dismissed cookie banner")
|
||||
await asyncio.sleep(1)
|
||||
except: pass
|
||||
|
||||
await self.log_listing_details(listing)
|
||||
# Try to handle consent manager (consentmanager.net)
|
||||
try:
|
||||
consent_selectors = [
|
||||
'#cmpbntyestxt', '.cmpboxbtnyes', 'a.cmpboxbtn.cmpboxbtnyes',
|
||||
'#cmpwelcomebtnyes', '.cmptxt_btn_yes'
|
||||
]
|
||||
for sel in consent_selectors:
|
||||
consent_btn = await page.query_selector(sel)
|
||||
if consent_btn and await consent_btn.is_visible():
|
||||
await consent_btn.click()
|
||||
logger.info("[HOWOGE] Dismissed consent manager")
|
||||
await asyncio.sleep(1)
|
||||
break
|
||||
except: pass
|
||||
|
||||
logger.info("[HOWOGE] Searching for application button...")
|
||||
# Look for "Besichtigung vereinbaren" button
|
||||
logger.info("[HOWOGE] Looking for 'Besichtigung vereinbaren' button...")
|
||||
selectors = [
|
||||
'a[href*="besichtigung-vereinbaren"]',
|
||||
'a:has-text("Besichtigung vereinbaren")',
|
||||
|
|
@ -50,32 +67,160 @@ class HowogeHandler(BaseHandler):
|
|||
apply_btn = None
|
||||
for sel in selectors:
|
||||
all_btns = await page.query_selector_all(sel)
|
||||
logger.debug(f"[HOWOGE] Selector '{sel}': {len(all_btns)} matches")
|
||||
logger.info(f"[HOWOGE] Selector '{sel}' found {len(all_btns)} matches")
|
||||
for btn in all_btns:
|
||||
try:
|
||||
if await btn.is_visible():
|
||||
apply_btn = btn
|
||||
logger.info(f"[HOWOGE] Found visible application button: {sel}")
|
||||
logger.info(f"[HOWOGE] Found visible button with selector '{sel}'")
|
||||
break
|
||||
except Exception as e:
|
||||
logger.debug(f"[HOWOGE] Button visibility error: {e}")
|
||||
except:
|
||||
pass
|
||||
if apply_btn:
|
||||
break
|
||||
|
||||
if apply_btn:
|
||||
# Scroll the button into view and click
|
||||
logger.info("[HOWOGE] Found application button, scrolling into view...")
|
||||
await apply_btn.scroll_into_view_if_needed()
|
||||
await asyncio.sleep(0.5)
|
||||
logger.info("[HOWOGE] Clicking button...")
|
||||
await apply_btn.click()
|
||||
await asyncio.sleep(2)
|
||||
result["success"] = True
|
||||
result["message"] = "Application submitted successfully."
|
||||
await asyncio.sleep(3)
|
||||
await page.wait_for_load_state("networkidle")
|
||||
logger.info("[HOWOGE] Clicked button, starting multi-step form process...")
|
||||
|
||||
max_steps = 6 # safety limit
|
||||
for step in range(1, max_steps + 1):
|
||||
logger.info(f"[HOWOGE] Processing step {step}")
|
||||
await page.evaluate("window.scrollBy(0, 300)")
|
||||
await asyncio.sleep(0.5)
|
||||
email_field = await page.query_selector('input[name*="email" i]')
|
||||
if email_field and await email_field.is_visible():
|
||||
logger.info("[HOWOGE] Email field is visible - form is ready!")
|
||||
break
|
||||
checkboxes = await page.query_selector_all('input[type="checkbox"]')
|
||||
clicked_checkbox = False
|
||||
for checkbox in checkboxes:
|
||||
try:
|
||||
if await checkbox.is_visible() and not await checkbox.is_checked():
|
||||
await checkbox.evaluate("el => el.click()")
|
||||
clicked_checkbox = True
|
||||
logger.info(f"[HOWOGE] Clicked checkbox in step {step}")
|
||||
await asyncio.sleep(0.5)
|
||||
except Exception as e:
|
||||
logger.debug(f"[HOWOGE] Checkbox click failed: {e}")
|
||||
if clicked_checkbox:
|
||||
await asyncio.sleep(1)
|
||||
screenshot_path = DATA_DIR / f"step{step}_{listing['id']}.png"
|
||||
await page.screenshot(path=str(screenshot_path), full_page=True)
|
||||
weiter_btns = await page.query_selector_all('button:has-text("Weiter")')
|
||||
weiter_clicked = False
|
||||
for btn in weiter_btns:
|
||||
try:
|
||||
if await btn.is_visible():
|
||||
await btn.click()
|
||||
weiter_clicked = True
|
||||
logger.info(f"[HOWOGE] Clicked 'Weiter' button in step {step}")
|
||||
await asyncio.sleep(2)
|
||||
await page.wait_for_load_state("networkidle")
|
||||
break
|
||||
except Exception as e:
|
||||
logger.debug(f"[HOWOGE] Weiter click failed: {e}")
|
||||
if not weiter_clicked and not clicked_checkbox:
|
||||
logger.warning(f"[HOWOGE] No action possible in step {step}, breaking")
|
||||
break
|
||||
|
||||
# Now try to fill the form
|
||||
logger.info("[HOWOGE] Attempting to fill form fields...")
|
||||
vorname_field = await page.query_selector('input[name*="firstName" i], input[name*="vorname" i]')
|
||||
nachname_field = await page.query_selector('input[name*="lastName" i], input[name*="nachname" i]')
|
||||
email_field = await page.query_selector('input[type="email"], input[name*="email" i]')
|
||||
form_filled = False
|
||||
if vorname_field and await vorname_field.is_visible():
|
||||
await vorname_field.fill(os.environ.get("FORM_VORNAME", "Max"))
|
||||
logger.info(f"[HOWOGE] Filled Vorname: {os.environ.get('FORM_VORNAME', 'Max')}")
|
||||
form_filled = True
|
||||
else:
|
||||
logger.warning("[HOWOGE] Vorname field not found or not visible")
|
||||
if nachname_field and await nachname_field.is_visible():
|
||||
await nachname_field.fill(os.environ.get("FORM_NACHNAME", "Mustermann"))
|
||||
logger.info(f"[HOWOGE] Filled Nachname: {os.environ.get('FORM_NACHNAME', 'Mustermann')}")
|
||||
form_filled = True
|
||||
else:
|
||||
logger.warning("[HOWOGE] Nachname field not found or not visible")
|
||||
if email_field and await email_field.is_visible():
|
||||
await email_field.fill(os.environ.get("FORM_EMAIL", "test@example.com"))
|
||||
logger.info(f"[HOWOGE] Filled Email: {os.environ.get('FORM_EMAIL', 'test@example.com')}")
|
||||
form_filled = True
|
||||
else:
|
||||
logger.warning("[HOWOGE] Email field not found or not visible")
|
||||
phone_field = await page.query_selector('input[type="tel"], input[name*="telefon" i], input[name*="phone" i]')
|
||||
if phone_field and await phone_field.is_visible():
|
||||
await phone_field.fill(os.environ.get("FORM_PHONE", "0123456789"))
|
||||
logger.info(f"[HOWOGE] Filled Phone: {os.environ.get('FORM_PHONE', '0123456789')}")
|
||||
screenshot_path2 = DATA_DIR / f"filled_{listing['id']}.png"
|
||||
await page.screenshot(path=str(screenshot_path2), full_page=True)
|
||||
logger.info(f"[HOWOGE] Saved filled form screenshot to {screenshot_path2}")
|
||||
if form_filled:
|
||||
submit_btn = None
|
||||
for selector in ['button:has-text("Anfrage senden")', 'button:has-text("Absenden")', 'button:has-text("Senden")']:
|
||||
btn = await page.query_selector(selector)
|
||||
if btn and await btn.is_visible():
|
||||
submit_btn = btn
|
||||
logger.info(f"[HOWOGE] Found submit button with selector: {selector}")
|
||||
break
|
||||
if submit_btn:
|
||||
logger.info("[HOWOGE] Found submit button, clicking...")
|
||||
await submit_btn.click()
|
||||
await asyncio.sleep(3)
|
||||
await page.wait_for_load_state("networkidle")
|
||||
screenshot_path3 = DATA_DIR / f"submitted_{listing['id']}.png"
|
||||
await page.screenshot(path=str(screenshot_path3))
|
||||
logger.info(f"[HOWOGE] Saved post-submit screenshot to {screenshot_path3}")
|
||||
content = await page.content()
|
||||
if "erfolgreich" in content.lower() or "gesendet" in content.lower() or "danke" in content.lower() or "bestätigung" in content.lower():
|
||||
result["success"] = True
|
||||
result["message"] = "Application submitted successfully"
|
||||
logger.info("[HOWOGE] Success! Confirmation message detected")
|
||||
else:
|
||||
result["success"] = False
|
||||
result["message"] = "Form submitted but no confirmation detected"
|
||||
logger.warning("[HOWOGE] Form submitted but no clear confirmation")
|
||||
else:
|
||||
result["success"] = False
|
||||
result["message"] = "Form filled but no submit button found"
|
||||
logger.warning("[HOWOGE] Could not find submit button")
|
||||
else:
|
||||
result["success"] = False
|
||||
result["message"] = "Could not find form fields to fill after navigating steps"
|
||||
logger.warning("[HOWOGE] No form fields found after multi-step navigation")
|
||||
else:
|
||||
logger.warning("[HOWOGE] No application button found.")
|
||||
result["message"] = "No application button found."
|
||||
result["message"] = "No application button found"
|
||||
logger.warning("[HOWOGE] Could not find 'Besichtigung vereinbaren' button")
|
||||
screenshot_path = DATA_DIR / f"nobtn_{listing['id']}.png"
|
||||
await page.screenshot(path=str(screenshot_path))
|
||||
buttons = await page.query_selector_all('button, a.btn, a[class*="button"]')
|
||||
for btn in buttons[:10]:
|
||||
try:
|
||||
text = await btn.inner_text()
|
||||
logger.info(f"[HOWOGE] Found button: {text[:50]}")
|
||||
except:
|
||||
pass
|
||||
except Exception as e:
|
||||
result["message"] = f"Error during application: {e}"
|
||||
logger.error(f"[HOWOGE] Application error: {e}")
|
||||
result["message"] = f"Error: {str(e)}"
|
||||
logger.error(f"[HOWOGE] Exception: {str(e)}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
# Save debug HTML on error
|
||||
debug_html_path = DATA_DIR / f"debug_{listing['id']}.html"
|
||||
try:
|
||||
html = await page.content()
|
||||
with open(debug_html_path, "w", encoding="utf-8") as f:
|
||||
f.write(html)
|
||||
logger.info(f"[HOWOGE] Saved debug HTML to {debug_html_path}")
|
||||
except Exception as html_e:
|
||||
logger.error(f"[HOWOGE] Failed to save debug HTML: {html_e}")
|
||||
finally:
|
||||
await page.close()
|
||||
|
||||
return result
|
||||
|
|
@ -1,6 +1,20 @@
|
|||
from .base_handler import BaseHandler
|
||||
|
||||
import logging
|
||||
import asyncio
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
# Load environment variables for form fields and data dir
|
||||
FORM_VORNAME = os.environ.get("FORM_VORNAME", "")
|
||||
FORM_NACHNAME = os.environ.get("FORM_NACHNAME", "")
|
||||
FORM_STRASSE = os.environ.get("FORM_STRASSE", "")
|
||||
FORM_HAUSNUMMER = os.environ.get("FORM_HAUSNUMMER", "")
|
||||
FORM_PLZ = os.environ.get("FORM_PLZ", "")
|
||||
FORM_ORT = os.environ.get("FORM_ORT", "")
|
||||
FORM_PHONE = os.environ.get("FORM_PHONE", "")
|
||||
FORM_EMAIL = os.environ.get("FORM_EMAIL", "")
|
||||
DATA_DIR = Path(os.environ.get("DATA_DIR", "data"))
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -11,77 +25,178 @@ class StadtUndLandHandler(BaseHandler):
|
|||
async def apply(self, listing: dict, result: dict) -> dict:
|
||||
page = await self.context.new_page()
|
||||
try:
|
||||
logger.info(f"[STADT UND LAND] Open: {listing['link']}")
|
||||
logger.info(f"[STADTUNDLAND] Opening page: {listing['link']}")
|
||||
await page.goto(listing["link"], wait_until="networkidle")
|
||||
logger.info("[STADTUNDLAND] Page loaded")
|
||||
await asyncio.sleep(2)
|
||||
|
||||
# Always handle cookies and consent before anything else
|
||||
await self.handle_cookies(page)
|
||||
await self.handle_consent(page)
|
||||
|
||||
# Save HTML after modal handling for debugging
|
||||
# Dismiss cookie banner
|
||||
try:
|
||||
html_content = await page.content()
|
||||
with open("data/stadtundland_debug.html", "w", encoding="utf-8") as f:
|
||||
f.write(html_content)
|
||||
cookie_btn = await page.query_selector('button:has-text("Akzeptieren"), button:has-text("Alle akzeptieren")')
|
||||
if cookie_btn and await cookie_btn.is_visible():
|
||||
await cookie_btn.click()
|
||||
logger.info("[STADTUNDLAND] Dismissed cookie banner")
|
||||
await asyncio.sleep(1)
|
||||
except Exception as e:
|
||||
logger.debug(f"[STADT UND LAND] Debug HTML not saved: {e}")
|
||||
logger.debug(f"[STADTUNDLAND] Cookie banner dismiss failed: {e}")
|
||||
|
||||
# 404/permanent fail detection
|
||||
error_texts = [
|
||||
"Hier ist etwas schief gelaufen",
|
||||
"Leider können wir Ihnen zur Zeit keine Details zu diesem Inserat anzeigen"
|
||||
]
|
||||
page_text = await page.text_content('body')
|
||||
if page_text:
|
||||
for err in error_texts:
|
||||
if err in page_text:
|
||||
logger.warning(f"[STADT UND LAND] Permanent fail: {err}")
|
||||
result["permanent_fail"] = True
|
||||
result["message"] = "Listing is no longer available (404 detected on STADT UND LAND)."
|
||||
await page.close()
|
||||
return result
|
||||
# Scroll to form
|
||||
await page.evaluate("window.scrollBy(0, 500)")
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# Look for application button (robust selectors)
|
||||
logger.info("[STADT UND LAND] Searching for application button...")
|
||||
selectors = [
|
||||
'a[href*="bewerben"]',
|
||||
'button:has-text("Bewerben")',
|
||||
'a:has-text("Bewerben")',
|
||||
'button.btn',
|
||||
'a.Button_button__JnZ4E',
|
||||
'button.Button_button__JnZ4E',
|
||||
]
|
||||
# Fill out the embedded form directly
|
||||
form_filled = False
|
||||
try:
|
||||
# Vorname
|
||||
vorname_field = await page.query_selector('input[name="name"]')
|
||||
if vorname_field and await vorname_field.is_visible():
|
||||
await vorname_field.fill(FORM_VORNAME)
|
||||
logger.info(f"[STADTUNDLAND] Filled Vorname: {FORM_VORNAME}")
|
||||
form_filled = True
|
||||
# Nachname
|
||||
nachname_field = await page.query_selector('input[name="surname"]')
|
||||
if nachname_field and await nachname_field.is_visible():
|
||||
await nachname_field.fill(FORM_NACHNAME)
|
||||
logger.info(f"[STADTUNDLAND] Filled Nachname: {FORM_NACHNAME}")
|
||||
form_filled = True
|
||||
# Straße
|
||||
street_field = await page.query_selector('input[name="street"]')
|
||||
if street_field and await street_field.is_visible():
|
||||
await street_field.fill(FORM_STRASSE)
|
||||
logger.info(f"[STADTUNDLAND] Filled Straße: {FORM_STRASSE}")
|
||||
form_filled = True
|
||||
# Hausnummer
|
||||
house_field = await page.query_selector('input[name="houseNo"]')
|
||||
if house_field and await house_field.is_visible():
|
||||
await house_field.fill(FORM_HAUSNUMMER)
|
||||
logger.info(f"[STADTUNDLAND] Filled Hausnummer: {FORM_HAUSNUMMER}")
|
||||
form_filled = True
|
||||
# PLZ
|
||||
plz_field = await page.query_selector('input[name="postalCode"]')
|
||||
if plz_field and await plz_field.is_visible():
|
||||
await plz_field.fill(FORM_PLZ)
|
||||
logger.info(f"[STADTUNDLAND] Filled PLZ: {FORM_PLZ}")
|
||||
form_filled = True
|
||||
# Ort
|
||||
city_field = await page.query_selector('input[name="city"]')
|
||||
if city_field and await city_field.is_visible():
|
||||
await city_field.fill(FORM_ORT)
|
||||
logger.info(f"[STADTUNDLAND] Filled Ort: {FORM_ORT}")
|
||||
form_filled = True
|
||||
# Telefon
|
||||
phone_field = await page.query_selector('input[name="phone"]')
|
||||
if phone_field and await phone_field.is_visible():
|
||||
await phone_field.fill(FORM_PHONE)
|
||||
logger.info(f"[STADTUNDLAND] Filled Telefon: {FORM_PHONE}")
|
||||
form_filled = True
|
||||
# E-Mail
|
||||
email_field = await page.query_selector('input[name="email"]')
|
||||
if email_field and await email_field.is_visible():
|
||||
await email_field.fill(FORM_EMAIL)
|
||||
logger.info(f"[STADTUNDLAND] Filled E-Mail: {FORM_EMAIL}")
|
||||
form_filled = True
|
||||
except Exception as e:
|
||||
logger.warning(f"[STADTUNDLAND] Error filling form fields: {e}")
|
||||
|
||||
apply_btn = None
|
||||
for sel in selectors:
|
||||
all_btns = await page.query_selector_all(sel)
|
||||
logger.debug(f"[STADT UND LAND] Selector '{sel}': {len(all_btns)} matches")
|
||||
for btn in all_btns:
|
||||
try:
|
||||
if await btn.is_visible():
|
||||
apply_btn = btn
|
||||
logger.info(f"[STADT UND LAND] Found visible application button: {sel}")
|
||||
break
|
||||
except Exception as e:
|
||||
logger.debug(f"[STADT UND LAND] Button visibility error: {e}")
|
||||
if apply_btn:
|
||||
break
|
||||
# Click privacy checkbox
|
||||
try:
|
||||
privacy_checkbox = await page.query_selector('input[name="privacy"]')
|
||||
if privacy_checkbox and await privacy_checkbox.is_visible():
|
||||
if not await privacy_checkbox.is_checked():
|
||||
await privacy_checkbox.click()
|
||||
logger.info("[STADTUNDLAND] Clicked privacy checkbox")
|
||||
except Exception as e:
|
||||
logger.warning(f"[STADTUNDLAND] Could not click privacy checkbox: {e}")
|
||||
|
||||
if apply_btn:
|
||||
await apply_btn.scroll_into_view_if_needed()
|
||||
await asyncio.sleep(0.5)
|
||||
await apply_btn.click()
|
||||
await asyncio.sleep(2)
|
||||
result["success"] = True
|
||||
result["message"] = "Application submitted successfully."
|
||||
# Click provision checkbox (optional)
|
||||
try:
|
||||
provision_checkbox = await page.query_selector('input[name="provision"]')
|
||||
if provision_checkbox and await provision_checkbox.is_visible():
|
||||
if not await provision_checkbox.is_checked():
|
||||
await provision_checkbox.click()
|
||||
logger.info("[STADTUNDLAND] Clicked provision checkbox")
|
||||
except Exception as e:
|
||||
logger.warning(f"[STADTUNDLAND] Could not click provision checkbox: {e}")
|
||||
|
||||
await asyncio.sleep(1)
|
||||
|
||||
# Screenshot after filling form
|
||||
screenshot_path2 = DATA_DIR / f"stadtundland_filled_{listing['id']}.png"
|
||||
await page.screenshot(path=str(screenshot_path2), full_page=True)
|
||||
logger.info(f"[STADTUNDLAND] Saved filled form screenshot to {screenshot_path2}")
|
||||
|
||||
# Submit form
|
||||
if form_filled:
|
||||
try:
|
||||
pruefen_btn = await page.query_selector('button:has-text("Eingaben prüfen")')
|
||||
if pruefen_btn and await pruefen_btn.is_visible():
|
||||
await pruefen_btn.click()
|
||||
logger.info("[STADTUNDLAND] Clicked 'Eingaben prüfen' button")
|
||||
await asyncio.sleep(2)
|
||||
await page.wait_for_load_state("networkidle")
|
||||
|
||||
# Screenshot after validation
|
||||
screenshot_path3 = DATA_DIR / f"stadtundland_validated_{listing['id']}.png"
|
||||
await page.screenshot(path=str(screenshot_path3), full_page=True)
|
||||
logger.info(f"[STADTUNDLAND] Saved validation screenshot to {screenshot_path3}")
|
||||
|
||||
# Final submit
|
||||
final_submit_selectors = [
|
||||
'button:has-text("Absenden")',
|
||||
'button:has-text("Senden")',
|
||||
'button:has-text("Anfrage senden")',
|
||||
'button:has-text("Bestätigen")',
|
||||
'button[type="submit"]',
|
||||
]
|
||||
final_btn = None
|
||||
for selector in final_submit_selectors:
|
||||
btn = await page.query_selector(selector)
|
||||
if btn and await btn.is_visible():
|
||||
final_btn = btn
|
||||
logger.info(f"[STADTUNDLAND] Found final submit button: {selector}")
|
||||
break
|
||||
if final_btn:
|
||||
await final_btn.click()
|
||||
logger.info("[STADTUNDLAND] Clicked final submit button")
|
||||
await asyncio.sleep(3)
|
||||
await page.wait_for_load_state("networkidle")
|
||||
|
||||
# Screenshot after submission
|
||||
screenshot_path4 = DATA_DIR / f"stadtundland_submitted_{listing['id']}.png"
|
||||
await page.screenshot(path=str(screenshot_path4), full_page=True)
|
||||
logger.info(f"[STADTUNDLAND] Saved submission screenshot to {screenshot_path4}")
|
||||
|
||||
# Check for confirmation
|
||||
content = await page.content()
|
||||
if any(word in content.lower() for word in ["erfolgreich", "gesendet", "danke", "bestätigung"]):
|
||||
result["success"] = True
|
||||
result["message"] = "Application submitted successfully"
|
||||
logger.info("[STADTUNDLAND] Success! Confirmation message detected")
|
||||
else:
|
||||
result["success"] = True
|
||||
result["message"] = "Form submitted"
|
||||
logger.info("[STADTUNDLAND] Form submitted")
|
||||
else:
|
||||
result["success"] = False
|
||||
result["message"] = "Validated but final submit button not found"
|
||||
logger.warning("[STADTUNDLAND] Final submit button not found")
|
||||
else:
|
||||
result["success"] = False
|
||||
result["message"] = "Form filled but 'Eingaben prüfen' button not found"
|
||||
logger.warning("[STADTUNDLAND] 'Eingaben prüfen' button not found")
|
||||
except Exception as e:
|
||||
result["success"] = False
|
||||
result["message"] = f"Submit error: {str(e)}"
|
||||
logger.warning(f"[STADTUNDLAND] Submit error: {e}")
|
||||
else:
|
||||
logger.warning("[STADT UND LAND] No application button found.")
|
||||
result["message"] = "No application button found."
|
||||
result["success"] = False
|
||||
result["message"] = "No form fields found on page"
|
||||
logger.warning("[STADTUNDLAND] Could not find form fields")
|
||||
|
||||
except Exception as e:
|
||||
result["message"] = f"Error during application: {e}"
|
||||
logger.error(f"[STADT UND LAND] Application error: {e}")
|
||||
result["success"] = False
|
||||
result["message"] = f"Error: {str(e)}"
|
||||
logger.error(f"[STADTUNDLAND] Exception: {str(e)}")
|
||||
finally:
|
||||
await page.close()
|
||||
|
||||
return result
|
||||
|
|
@ -119,8 +119,47 @@ class WBMHandler(BaseHandler):
|
|||
logger.info("[WBM] Clicking application button...")
|
||||
await apply_btn.click()
|
||||
await asyncio.sleep(2)
|
||||
result["success"] = True
|
||||
result["message"] = "Application button clicked on detail page. (Submission not implemented)"
|
||||
# --- Post-click confirmation logic ---
|
||||
logger.info("[WBM] Clicked application button, checking for confirmation...")
|
||||
# Save screenshot and HTML after click
|
||||
try:
|
||||
await page.screenshot(path="data/wbm_after_apply.png")
|
||||
logger.info("[WBM] Saved screenshot after application click.")
|
||||
except Exception as e:
|
||||
logger.warning(f"[WBM] Could not save screenshot: {e}")
|
||||
try:
|
||||
html_after = await page.content()
|
||||
with open("data/wbm_after_apply.html", "w", encoding="utf-8") as f:
|
||||
f.write(html_after)
|
||||
logger.info("[WBM] Saved HTML after application click.")
|
||||
except Exception as e:
|
||||
logger.warning(f"[WBM] Could not save HTML after apply: {e}")
|
||||
|
||||
# Look for confirmation message on the page
|
||||
confirmation_selectors = [
|
||||
'text="Vielen Dank"',
|
||||
'text="Ihre Anfrage wurde gesendet"',
|
||||
'text="Bestätigung"',
|
||||
'div:has-text("Vielen Dank")',
|
||||
'div:has-text("Ihre Anfrage wurde gesendet")',
|
||||
]
|
||||
confirmed = False
|
||||
for sel in confirmation_selectors:
|
||||
try:
|
||||
el = await page.query_selector(sel)
|
||||
if el and await el.is_visible():
|
||||
logger.info(f"[WBM] Found confirmation element: {sel}")
|
||||
confirmed = True
|
||||
break
|
||||
except Exception as e:
|
||||
logger.debug(f"[WBM] Error checking confirmation selector {sel}: {e}")
|
||||
if confirmed:
|
||||
result["success"] = True
|
||||
result["message"] = "Application submitted and confirmation detected."
|
||||
else:
|
||||
logger.warning("[WBM] No confirmation message detected after application click.")
|
||||
result["success"] = False
|
||||
result["message"] = "Clicked application button, but no confirmation detected. Check screenshot and HTML."
|
||||
else:
|
||||
result["message"] = "No application button found on detail page."
|
||||
except Exception as e:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue