roughly working again, now dev docker exists

handlers/gewobag_handler.py

@@ -5,23 +5,49 @@ import asyncio
 logger = logging.getLogger(__name__)
 
 class GewobagHandler(BaseHandler):
+    def __init__(self, browser_context):
+        self.context = browser_context
+
     async def apply(self, listing: dict, result: dict) -> dict:
         page = await self.context.new_page()
         try:
             logger.info(f"[GEWOBAG] Opening page: {listing['link']}")
-            await page.goto(listing["link"], wait_until="networkidle")
+            response = await page.goto(listing["link"], wait_until="networkidle")
             logger.info("[GEWOBAG] Page loaded")
             await asyncio.sleep(2)
 
-            # Handle cookies and consent
+            # Detect 404 by status or page title
+            status = response.status if response else None
+            page_title = await page.title()
+            if status == 404 or (page_title and "404" in page_title):
+                logger.warning(f"[GEWOBAG] Listing is down (404): {listing['link']}")
+                result["success"] = False
+                result["message"] = "Listing is no longer available (404). Application impossible. Will not retry."
+                result["permanent_fail"] = True
+                return result
+
+            # Always handle cookies and consent before anything else
             await self.handle_cookies(page)
             await self.handle_consent(page)
 
-            # Look for application button
+            # Save HTML after modal handling for debugging
+            try:
+                html_content = await page.content()
+                with open("data/gewobag_debug.html", "w", encoding="utf-8") as f:
+                    f.write(html_content)
+            except Exception as e:
+                logger.warning(f"[GEWOBAG] Could not save debug HTML: {e}")
+
+            # Log listing details
+            await self.log_listing_details(listing)
+
+            # Look for application button ("Anfrage senden") in tab or footer
             logger.info("[GEWOBAG] Looking for application button...")
             selectors = [
-                'a[href*="bewerben"]',
-                'button:has-text("Bewerben")'
+                'button.rental-contact',
+                'button:has-text("Anfrage senden")',
+                'div.contact-button button',
+                'iframe#contact-iframe',
             ]
 
             apply_btn = None
@@ -39,6 +65,24 @@ class GewobagHandler(BaseHandler):
                 if apply_btn:
                     break
 
+            # If not found, check for iframe (Wohnungshelden)
+            if not apply_btn:
+                iframe = await page.query_selector('iframe#contact-iframe')
+                if iframe:
+                    logger.info("[GEWOBAG] Found Wohnungshelden iframe, switching context...")
+                    frame = await iframe.content_frame()
+                    if frame:
+                        # Try to find a submit/apply button in the iframe
+                        iframe_btns = await frame.query_selector_all('button, input[type="submit"]')
+                        for btn in iframe_btns:
+                            try:
+                                if await btn.is_visible():
+                                    apply_btn = btn
+                                    logger.info("[GEWOBAG] Found visible button in iframe")
+                                    break
+                            except Exception as e:
+                                logger.warning(f"[GEWOBAG] Error checking iframe button visibility: {e}")
+
             if apply_btn:
                 logger.info("[GEWOBAG] Found application button, scrolling into view...")
                 await apply_btn.scroll_into_view_if_needed()