working app
This commit is contained in:
parent
8e69e30387
commit
3057cda8d3
12 changed files with 708 additions and 232 deletions
|
|
@ -9,10 +9,15 @@ class HowogeHandler(BaseHandler):
|
|||
self.context = browser_context
|
||||
|
||||
async def apply(self, listing: dict, result: dict) -> dict:
|
||||
import os
|
||||
from pathlib import Path
|
||||
DATA_DIR = Path("data/howoge")
|
||||
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
page = await self.context.new_page()
|
||||
try:
|
||||
logger.info(f"[HOWOGE] Open: {listing['link']}")
|
||||
logger.info(f"[HOWOGE] Opening page: {listing['link']}")
|
||||
response = await page.goto(listing["link"], wait_until="networkidle")
|
||||
logger.info("[HOWOGE] Page loaded")
|
||||
await asyncio.sleep(2)
|
||||
|
||||
# Detect 404 by status or page title
|
||||
|
|
@ -23,23 +28,35 @@ class HowogeHandler(BaseHandler):
|
|||
result["success"] = False
|
||||
result["message"] = "Listing is no longer available (404). Application impossible. Will not retry."
|
||||
result["permanent_fail"] = True
|
||||
await page.close()
|
||||
return result
|
||||
|
||||
# Always handle cookies and consent before anything else
|
||||
await self.handle_cookies(page)
|
||||
await self.handle_consent(page)
|
||||
|
||||
# Save HTML after modal handling for debugging
|
||||
# Handle cookies
|
||||
try:
|
||||
html_content = await page.content()
|
||||
with open("data/howoge_debug.html", "w", encoding="utf-8") as f:
|
||||
f.write(html_content)
|
||||
except Exception as e:
|
||||
logger.debug(f"[HOWOGE] Debug HTML not saved: {e}")
|
||||
cookie_btn = await page.query_selector('button:has-text("Akzeptieren"), button:has-text("Alle akzeptieren")')
|
||||
if cookie_btn and await cookie_btn.is_visible():
|
||||
await cookie_btn.click()
|
||||
logger.info("[HOWOGE] Dismissed cookie banner")
|
||||
await asyncio.sleep(1)
|
||||
except: pass
|
||||
|
||||
await self.log_listing_details(listing)
|
||||
# Try to handle consent manager (consentmanager.net)
|
||||
try:
|
||||
consent_selectors = [
|
||||
'#cmpbntyestxt', '.cmpboxbtnyes', 'a.cmpboxbtn.cmpboxbtnyes',
|
||||
'#cmpwelcomebtnyes', '.cmptxt_btn_yes'
|
||||
]
|
||||
for sel in consent_selectors:
|
||||
consent_btn = await page.query_selector(sel)
|
||||
if consent_btn and await consent_btn.is_visible():
|
||||
await consent_btn.click()
|
||||
logger.info("[HOWOGE] Dismissed consent manager")
|
||||
await asyncio.sleep(1)
|
||||
break
|
||||
except: pass
|
||||
|
||||
logger.info("[HOWOGE] Searching for application button...")
|
||||
# Look for "Besichtigung vereinbaren" button
|
||||
logger.info("[HOWOGE] Looking for 'Besichtigung vereinbaren' button...")
|
||||
selectors = [
|
||||
'a[href*="besichtigung-vereinbaren"]',
|
||||
'a:has-text("Besichtigung vereinbaren")',
|
||||
|
|
@ -50,32 +67,160 @@ class HowogeHandler(BaseHandler):
|
|||
apply_btn = None
|
||||
for sel in selectors:
|
||||
all_btns = await page.query_selector_all(sel)
|
||||
logger.debug(f"[HOWOGE] Selector '{sel}': {len(all_btns)} matches")
|
||||
logger.info(f"[HOWOGE] Selector '{sel}' found {len(all_btns)} matches")
|
||||
for btn in all_btns:
|
||||
try:
|
||||
if await btn.is_visible():
|
||||
apply_btn = btn
|
||||
logger.info(f"[HOWOGE] Found visible application button: {sel}")
|
||||
logger.info(f"[HOWOGE] Found visible button with selector '{sel}'")
|
||||
break
|
||||
except Exception as e:
|
||||
logger.debug(f"[HOWOGE] Button visibility error: {e}")
|
||||
except:
|
||||
pass
|
||||
if apply_btn:
|
||||
break
|
||||
|
||||
if apply_btn:
|
||||
# Scroll the button into view and click
|
||||
logger.info("[HOWOGE] Found application button, scrolling into view...")
|
||||
await apply_btn.scroll_into_view_if_needed()
|
||||
await asyncio.sleep(0.5)
|
||||
logger.info("[HOWOGE] Clicking button...")
|
||||
await apply_btn.click()
|
||||
await asyncio.sleep(2)
|
||||
result["success"] = True
|
||||
result["message"] = "Application submitted successfully."
|
||||
await asyncio.sleep(3)
|
||||
await page.wait_for_load_state("networkidle")
|
||||
logger.info("[HOWOGE] Clicked button, starting multi-step form process...")
|
||||
|
||||
max_steps = 6 # safety limit
|
||||
for step in range(1, max_steps + 1):
|
||||
logger.info(f"[HOWOGE] Processing step {step}")
|
||||
await page.evaluate("window.scrollBy(0, 300)")
|
||||
await asyncio.sleep(0.5)
|
||||
email_field = await page.query_selector('input[name*="email" i]')
|
||||
if email_field and await email_field.is_visible():
|
||||
logger.info("[HOWOGE] Email field is visible - form is ready!")
|
||||
break
|
||||
checkboxes = await page.query_selector_all('input[type="checkbox"]')
|
||||
clicked_checkbox = False
|
||||
for checkbox in checkboxes:
|
||||
try:
|
||||
if await checkbox.is_visible() and not await checkbox.is_checked():
|
||||
await checkbox.evaluate("el => el.click()")
|
||||
clicked_checkbox = True
|
||||
logger.info(f"[HOWOGE] Clicked checkbox in step {step}")
|
||||
await asyncio.sleep(0.5)
|
||||
except Exception as e:
|
||||
logger.debug(f"[HOWOGE] Checkbox click failed: {e}")
|
||||
if clicked_checkbox:
|
||||
await asyncio.sleep(1)
|
||||
screenshot_path = DATA_DIR / f"step{step}_{listing['id']}.png"
|
||||
await page.screenshot(path=str(screenshot_path), full_page=True)
|
||||
weiter_btns = await page.query_selector_all('button:has-text("Weiter")')
|
||||
weiter_clicked = False
|
||||
for btn in weiter_btns:
|
||||
try:
|
||||
if await btn.is_visible():
|
||||
await btn.click()
|
||||
weiter_clicked = True
|
||||
logger.info(f"[HOWOGE] Clicked 'Weiter' button in step {step}")
|
||||
await asyncio.sleep(2)
|
||||
await page.wait_for_load_state("networkidle")
|
||||
break
|
||||
except Exception as e:
|
||||
logger.debug(f"[HOWOGE] Weiter click failed: {e}")
|
||||
if not weiter_clicked and not clicked_checkbox:
|
||||
logger.warning(f"[HOWOGE] No action possible in step {step}, breaking")
|
||||
break
|
||||
|
||||
# Now try to fill the form
|
||||
logger.info("[HOWOGE] Attempting to fill form fields...")
|
||||
vorname_field = await page.query_selector('input[name*="firstName" i], input[name*="vorname" i]')
|
||||
nachname_field = await page.query_selector('input[name*="lastName" i], input[name*="nachname" i]')
|
||||
email_field = await page.query_selector('input[type="email"], input[name*="email" i]')
|
||||
form_filled = False
|
||||
if vorname_field and await vorname_field.is_visible():
|
||||
await vorname_field.fill(os.environ.get("FORM_VORNAME", "Max"))
|
||||
logger.info(f"[HOWOGE] Filled Vorname: {os.environ.get('FORM_VORNAME', 'Max')}")
|
||||
form_filled = True
|
||||
else:
|
||||
logger.warning("[HOWOGE] Vorname field not found or not visible")
|
||||
if nachname_field and await nachname_field.is_visible():
|
||||
await nachname_field.fill(os.environ.get("FORM_NACHNAME", "Mustermann"))
|
||||
logger.info(f"[HOWOGE] Filled Nachname: {os.environ.get('FORM_NACHNAME', 'Mustermann')}")
|
||||
form_filled = True
|
||||
else:
|
||||
logger.warning("[HOWOGE] Nachname field not found or not visible")
|
||||
if email_field and await email_field.is_visible():
|
||||
await email_field.fill(os.environ.get("FORM_EMAIL", "test@example.com"))
|
||||
logger.info(f"[HOWOGE] Filled Email: {os.environ.get('FORM_EMAIL', 'test@example.com')}")
|
||||
form_filled = True
|
||||
else:
|
||||
logger.warning("[HOWOGE] Email field not found or not visible")
|
||||
phone_field = await page.query_selector('input[type="tel"], input[name*="telefon" i], input[name*="phone" i]')
|
||||
if phone_field and await phone_field.is_visible():
|
||||
await phone_field.fill(os.environ.get("FORM_PHONE", "0123456789"))
|
||||
logger.info(f"[HOWOGE] Filled Phone: {os.environ.get('FORM_PHONE', '0123456789')}")
|
||||
screenshot_path2 = DATA_DIR / f"filled_{listing['id']}.png"
|
||||
await page.screenshot(path=str(screenshot_path2), full_page=True)
|
||||
logger.info(f"[HOWOGE] Saved filled form screenshot to {screenshot_path2}")
|
||||
if form_filled:
|
||||
submit_btn = None
|
||||
for selector in ['button:has-text("Anfrage senden")', 'button:has-text("Absenden")', 'button:has-text("Senden")']:
|
||||
btn = await page.query_selector(selector)
|
||||
if btn and await btn.is_visible():
|
||||
submit_btn = btn
|
||||
logger.info(f"[HOWOGE] Found submit button with selector: {selector}")
|
||||
break
|
||||
if submit_btn:
|
||||
logger.info("[HOWOGE] Found submit button, clicking...")
|
||||
await submit_btn.click()
|
||||
await asyncio.sleep(3)
|
||||
await page.wait_for_load_state("networkidle")
|
||||
screenshot_path3 = DATA_DIR / f"submitted_{listing['id']}.png"
|
||||
await page.screenshot(path=str(screenshot_path3))
|
||||
logger.info(f"[HOWOGE] Saved post-submit screenshot to {screenshot_path3}")
|
||||
content = await page.content()
|
||||
if "erfolgreich" in content.lower() or "gesendet" in content.lower() or "danke" in content.lower() or "bestätigung" in content.lower():
|
||||
result["success"] = True
|
||||
result["message"] = "Application submitted successfully"
|
||||
logger.info("[HOWOGE] Success! Confirmation message detected")
|
||||
else:
|
||||
result["success"] = False
|
||||
result["message"] = "Form submitted but no confirmation detected"
|
||||
logger.warning("[HOWOGE] Form submitted but no clear confirmation")
|
||||
else:
|
||||
result["success"] = False
|
||||
result["message"] = "Form filled but no submit button found"
|
||||
logger.warning("[HOWOGE] Could not find submit button")
|
||||
else:
|
||||
result["success"] = False
|
||||
result["message"] = "Could not find form fields to fill after navigating steps"
|
||||
logger.warning("[HOWOGE] No form fields found after multi-step navigation")
|
||||
else:
|
||||
logger.warning("[HOWOGE] No application button found.")
|
||||
result["message"] = "No application button found."
|
||||
result["message"] = "No application button found"
|
||||
logger.warning("[HOWOGE] Could not find 'Besichtigung vereinbaren' button")
|
||||
screenshot_path = DATA_DIR / f"nobtn_{listing['id']}.png"
|
||||
await page.screenshot(path=str(screenshot_path))
|
||||
buttons = await page.query_selector_all('button, a.btn, a[class*="button"]')
|
||||
for btn in buttons[:10]:
|
||||
try:
|
||||
text = await btn.inner_text()
|
||||
logger.info(f"[HOWOGE] Found button: {text[:50]}")
|
||||
except:
|
||||
pass
|
||||
except Exception as e:
|
||||
result["message"] = f"Error during application: {e}"
|
||||
logger.error(f"[HOWOGE] Application error: {e}")
|
||||
result["message"] = f"Error: {str(e)}"
|
||||
logger.error(f"[HOWOGE] Exception: {str(e)}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
# Save debug HTML on error
|
||||
debug_html_path = DATA_DIR / f"debug_{listing['id']}.html"
|
||||
try:
|
||||
html = await page.content()
|
||||
with open(debug_html_path, "w", encoding="utf-8") as f:
|
||||
f.write(html)
|
||||
logger.info(f"[HOWOGE] Saved debug HTML to {debug_html_path}")
|
||||
except Exception as html_e:
|
||||
logger.error(f"[HOWOGE] Failed to save debug HTML: {html_e}")
|
||||
finally:
|
||||
await page.close()
|
||||
|
||||
return result
|
||||
Loading…
Add table
Add a link
Reference in a new issue