From 540a3cc8846d4a63d2a0f4d28ecdda9ce0c294db Mon Sep 17 00:00:00 2001 From: Aron Date: Wed, 31 Dec 2025 16:06:42 +0100 Subject: [PATCH] mostly working shape? --- application_handler.py | 259 +++++++++++++++++++++--------- docker-compose.dev.yml | 2 +- handlers/degewo_handler.py | 155 +++++++++++++++++- handlers/wgcompany_notifier.py | 1 + requirements.txt | 1 + telegram_bot.py | 191 +++++++++++----------- tests/test_application_handler.py | 14 +- tests/test_company_detection.py | 3 +- tests/test_error_rate_plot.py | 14 +- tests/test_handlers.py | 5 +- 10 files changed, 462 insertions(+), 183 deletions(-) diff --git a/application_handler.py b/application_handler.py index e4b22bf..013ed7e 100644 --- a/application_handler.py +++ b/application_handler.py @@ -108,7 +108,8 @@ class ApplicationHandler: # Send via TelegramBot if available if hasattr(self, 'telegram_bot') and self.telegram_bot: logger.info(f"Notifying Telegram: {listing['address']} ({listing['rooms']}, {listing['size']}, {listing['price']})") - self.telegram_bot._send_message(message) + loop = getattr(self.telegram_bot, 'event_loop', None) or asyncio.get_event_loop() + asyncio.run_coroutine_threadsafe(self.telegram_bot._send_message(message), loop) else: logger.info(f"[TELEGRAM] Would send message for: {listing['address']} ({listing['rooms']}, {listing['size']}, {listing['price']})") @@ -313,69 +314,124 @@ class ApplicationHandler: def _generate_weekly_plot(self) -> str: - """Generate a heatmap of listings by day of week and hour. Always returns a plot path, even if no data.""" + """Generate a heatmap, bar chart, line chart, and summary of listings by day/hour, like monitor.py.""" plot_path = DATA_DIR / "weekly_plot.png" try: if not TIMING_FILE.exists(): - logger.warning("No timing file found for weekly plot. Generating empty plot.") - # Generate empty plot - fig, ax = plt.subplots(figsize=(10, 6)) - ax.set_xticks(range(24)) - ax.set_yticks(range(7)) - ax.set_xticklabels([f"{h}:00" for h in range(24)], rotation=90) - ax.set_yticklabels(["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]) - ax.set_title("Listings Heatmap (No Data)") - ax.text(0.5, 0.5, "No data available", fontsize=18, ha='center', va='center', transform=ax.transAxes, color='gray') - plt.savefig(plot_path) - plt.close(fig) - return str(plot_path) + logger.warning("No timing data file found") + return "" - df = pd.read_csv(TIMING_FILE, parse_dates=["timestamp"]) - if df.empty: - logger.warning("Timing file is empty. Generating empty plot.") - fig, ax = plt.subplots(figsize=(10, 6)) - ax.set_xticks(range(24)) - ax.set_yticks(range(7)) - ax.set_xticklabels([f"{h}:00" for h in range(24)], rotation=90) - ax.set_yticklabels(["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]) - ax.set_title("Listings Heatmap (No Data)") - ax.text(0.5, 0.5, "No data available", fontsize=18, ha='center', va='center', transform=ax.transAxes, color='gray') - plt.savefig(plot_path) - plt.close(fig) - return str(plot_path) + df = pd.read_csv(TIMING_FILE) + if len(df) < 1: + logger.warning("Timing file is empty") + return "" - df["day_of_week"] = df["timestamp"].dt.dayofweek - df["hour"] = df["timestamp"].dt.hour - heatmap_data = df.groupby(["day_of_week", "hour"]).size().unstack(fill_value=0) + logger.info(f"Loaded {len(df)} listing records for plot") - fig, ax = plt.subplots(figsize=(10, 6)) - cax = ax.matshow(heatmap_data, cmap="YlGnBu", aspect="auto") - fig.colorbar(cax) + # Create day-hour matrix + days_order = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'] - ax.set_xticks(range(24)) - ax.set_yticks(range(7)) - ax.set_xticklabels([f"{h}:00" for h in range(24)], rotation=90) - ax.set_yticklabels(["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]) + # Count listings per day and hour + heatmap_data = pd.DataFrame(0, index=days_order, columns=range(24)) - ax.set_title("Listings Heatmap (Day of Week vs Hour)") + for _, row in df.iterrows(): + day = row['weekday'] + hour = int(row['hour']) + if day in days_order: + # Use pd.to_numeric to ensure value is numeric before incrementing + val = pd.to_numeric(heatmap_data.loc[day, hour], errors='coerce') + if pd.isna(val): + heatmap_data.loc[day, hour] = 1 + else: + heatmap_data.loc[day, hour] = int(val) + 1 - plt.savefig(plot_path) - plt.close(fig) - logger.info(f"Weekly plot saved to {plot_path}") + # Create figure with two subplots + fig, axes = plt.subplots(2, 2, figsize=(14, 10)) + fig.suptitle('Listing Appearance Patterns', fontsize=16, fontweight='bold') + + # 1. Heatmap - Day vs Hour + ax1 = axes[0, 0] + im = ax1.imshow(heatmap_data.values, cmap='YlOrRd', aspect='auto') + ax1.set_xticks(range(24)) + ax1.set_xticklabels(range(24), fontsize=8) + ax1.set_yticks(range(7)) + ax1.set_yticklabels(days_order) + ax1.set_xlabel('Hour of Day') + ax1.set_ylabel('Day of Week') + ax1.set_title('Listings by Day & Hour') + plt.colorbar(im, ax=ax1, label='Count') + + # 2. Bar chart - By day of week + ax2 = axes[0, 1] + day_counts = df['weekday'].value_counts().reindex(days_order, fill_value=0) + colors = plt.cm.get_cmap('Blues')(day_counts / day_counts.max() if day_counts.max() > 0 else day_counts) + bars = ax2.bar(range(7), day_counts.values, color=colors) + ax2.set_xticks(range(7)) + ax2.set_xticklabels(['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']) + ax2.set_xlabel('Day of Week') + ax2.set_ylabel('Number of Listings') + ax2.set_title('Total Listings by Day') + for i, v in enumerate(day_counts.values): + if v > 0: + ax2.text(i, v + 0.1, str(v), ha='center', fontsize=9) + + # 3. Line chart - By hour + ax3 = axes[1, 0] + hour_counts = df['hour'].value_counts().reindex(range(24), fill_value=0) + ax3.plot(range(24), hour_counts.values, marker='o', linewidth=2, markersize=4, color='#2E86AB') + ax3.fill_between(range(24), hour_counts.values, alpha=0.3, color='#2E86AB') + ax3.set_xticks(range(0, 24, 2)) + ax3.set_xlabel('Hour of Day') + ax3.set_ylabel('Number of Listings') + ax3.set_title('Total Listings by Hour') + ax3.grid(True, alpha=0.3) + + # 4. Summary stats + ax4 = axes[1, 1] + ax4.axis('off') + + # Calculate best times + best_day = day_counts.idxmax() if day_counts.max() > 0 else "N/A" + best_hour = hour_counts.idxmax() if hour_counts.max() > 0 else "N/A" + total_listings = len(df) + + # Find peak combinations + peak_combo = heatmap_data.stack().idxmax() if heatmap_data.values.max() > 0 else ("N/A", "N/A") + + # Fix: Ensure peak_combo is iterable + if isinstance(peak_combo, tuple) and len(peak_combo) == 2: + stats_text = f"🎯 Peak time: {peak_combo[0]} at {peak_combo[1]}:00" + else: + stats_text = "🎯 Peak time: N/A" + + stats_text = f"""πŸ“Š Summary Statistics + +Total listings tracked: {total_listings} + +πŸ† Best day: {best_day} +⏰ Best hour: {best_hour}:00 +{stats_text} + +πŸ“ˆ Average per day: {total_listings/7:.1f} +πŸ“… Data collection period: + From: {df['timestamp'].min()[:10] if 'timestamp' in df.columns else 'N/A'} + To: {df['timestamp'].max()[:10] if 'timestamp' in df.columns else 'N/A'} +""" + ax4.text(0.1, 0.9, stats_text, transform=ax4.transAxes, fontsize=11, + verticalalignment='top', fontfamily='monospace', + bbox=dict(boxstyle='round', facecolor='wheat', alpha=0.5)) + + plt.tight_layout() + + # Save plot + plt.savefig(plot_path, dpi=150, bbox_inches='tight') + plt.close() + + logger.info(f"Plot saved to {plot_path}") return str(plot_path) except Exception as e: - logger.error(f"Failed to generate weekly plot: {e}") - # Always generate a fallback empty plot - fig, ax = plt.subplots(figsize=(10, 6)) - ax.set_xticks(range(24)) - ax.set_yticks(range(7)) - ax.set_xticklabels([f"{h}:00" for h in range(24)], rotation=90) - ax.set_yticklabels(["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]) - ax.set_title("Listings Heatmap (Error)") - ax.text(0.5, 0.5, "Plot error", fontsize=18, ha='center', va='center', transform=ax.transAxes, color='red') - plt.savefig(plot_path) - plt.close(fig) - return str(plot_path) + logger.error(f"Error creating plot: {e}") + return "" def _generate_error_rate_plot(self): @@ -383,6 +439,8 @@ class ApplicationHandler: Returns (plot_path, summary_text) or (None, "") if insufficient data. """ + import matplotlib.dates as mdates + from pathlib import Path if not self.applications_file.exists(): logger.warning("No applications.json found for errorrate plot") return None, "" @@ -390,25 +448,21 @@ class ApplicationHandler: try: with open(self.applications_file, 'r', encoding='utf-8') as f: apps = json.load(f) - if not apps: - logger.warning("No application data available for errorrate plot") return None, "" # Convert to DataFrame rows = [] for _id, rec in apps.items(): - rows.append({ - "id": _id, - "ts": pd.to_datetime(rec.get("timestamp")), - "success": rec.get("success", False), - "company": rec.get("company", "unknown") - }) - + ts = rec.get('timestamp') + try: + dt = pd.to_datetime(ts) + except Exception: + dt = pd.NaT + rows.append({'id': _id, 'company': rec.get('company'), 'success': bool(rec.get('success')), 'ts': dt}) df = pd.DataFrame(rows) df = df.dropna(subset=['ts']) if df.empty: - logger.warning("No valid data for errorrate plot") return None, "" df['date'] = df['ts'].dt.floor('D') @@ -419,28 +473,83 @@ class ApplicationHandler: # Ensure index is sorted by date for plotting grouped = grouped.sort_index() - # Prepare plot - fig, ax = plt.subplots(figsize=(10, 6)) - ax.plot(grouped.index, grouped['error_rate'], marker='o', color='red', label='Error Rate') - ax.set_title('Autopilot Error Rate Over Time') - ax.set_xlabel('Date') - ax.set_ylabel('Error Rate') - ax.legend() - ax.grid(True) + # Prepare plot: convert dates to matplotlib numeric x-values so bars and line align + fig, (ax1, ax2, ax3) = plt.subplots(3, 1, figsize=(12, 12), sharex=True) - # Save plot to the same directory as the applications file + dates = pd.to_datetime(grouped.index).to_pydatetime() + x = mdates.date2num(dates) + width = 0.6 # width in days for bars + + successes = grouped['successes'].values + failures = grouped['failures'].values + + ax1.bar(x, successes, width=width, color='#2E8B57', align='center') + ax1.bar(x, failures, bottom=successes, width=width, color='#C44A4A', align='center') + ax1.set_ylabel('Count') + ax1.set_title('Autopilot: Successes vs Failures (by day)') + ax1.set_xticks(x) + ax1.set_xlim(min(x) - 1, max(x) + 1) + ax1.xaxis.set_major_locator(mdates.AutoDateLocator()) + ax1.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d')) + + # Plot error rate line on same x (date) axis + ax2.plot(x, grouped['error_rate'].values, marker='o', color='#3333AA', linewidth=2) + ax2.set_ylim(-0.02, 1.02) + ax2.set_ylabel('Error rate') + ax2.set_xlabel('Date') + ax2.set_title('Daily Error Rate (failures / total)') + ax2.grid(True, alpha=0.3) + ax2.set_xticks(x) + ax2.set_xlim(min(x) - 1, max(x) + 1) + ax2.xaxis.set_major_locator(mdates.AutoDateLocator()) + ax2.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d')) + + # Error rate by company (line plot) + company_grouped = df.groupby(['date', 'company']).agg(total=('id','count'), successes=('success', lambda x: x.sum())) + company_grouped['failures'] = company_grouped['total'] - company_grouped['successes'] + company_grouped['error_rate'] = company_grouped['failures'] / company_grouped['total'] + company_grouped = company_grouped.reset_index() + error_rate_pivot = company_grouped.pivot(index='date', columns='company', values='error_rate') + for company in error_rate_pivot.columns: + y = error_rate_pivot[company].values + ax3.plot(x, y, marker='o', label=str(company)) + ax3.set_ylim(-0.02, 1.02) + ax3.set_ylabel('Error rate') + ax3.set_xlabel('Date') + ax3.set_title('Daily Error Rate by Company') + ax3.grid(True, alpha=0.3) + ax3.set_xticks(x) + ax3.set_xlim(min(x) - 1, max(x) + 1) + ax3.xaxis.set_major_locator(mdates.AutoDateLocator()) + ax3.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d')) + ax3.legend(title='Company', loc='upper right', fontsize='small') + + fig.autofmt_xdate() + plt.tight_layout() plot_path = self.applications_file.parent / 'error_rate.png' - plt.savefig(plot_path) + tmp_path = self.applications_file.parent / 'error_rate.tmp.png' + # Save to a temp file first and atomically replace to ensure overwrite + fig.savefig(tmp_path, format='png') plt.close(fig) + try: + tmp_path.replace(plot_path) + except Exception: + # Fallback: try removing existing and renaming + try: + if plot_path.exists(): + plot_path.unlink() + tmp_path.rename(plot_path) + except Exception: + logger.exception(f"Failed to write plot to {plot_path}") # Summary total_attempts = int(grouped['total'].sum()) total_success = int(grouped['successes'].sum()) total_fail = int(grouped['failures'].sum()) - overall_error = (total_fail / total_attempts) if total_attempts > 0 else 0.0 + overall_error = (total_fail / total_attempts) if total_attempts>0 else 0.0 summary = f"Total attempts: {total_attempts}\nSuccesses: {total_success}\nFailures: {total_fail}\nOverall error rate: {overall_error:.1%}" - return plot_path, summary + return str(plot_path), summary except Exception as e: logger.exception(f"Failed to generate error rate plot: {e}") return None, "" diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 09fad43..90b059b 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -11,5 +11,5 @@ services: volumes: - ./data:/app/data:rw environment: - - CHECK_INTERVAL=30 + - CHECK_INTERVAL=60 - WOHNBOT_DEV=1 diff --git a/handlers/degewo_handler.py b/handlers/degewo_handler.py index a2d6a1f..5bef474 100644 --- a/handlers/degewo_handler.py +++ b/handlers/degewo_handler.py @@ -12,6 +12,7 @@ class DegewoHandler(BaseHandler): self.context = browser_context async def apply(self, listing: dict, result: dict) -> dict: + import os DATA_DIR = Path("data/degewo") DATA_DIR.mkdir(parents=True, exist_ok=True) page = await self.context.new_page() @@ -60,31 +61,173 @@ class DegewoHandler(BaseHandler): await asyncio.sleep(3) # Degewo uses Wohnungshelden iframe for the application form - # Find the iframe and get its URL to navigate directly iframe_element = await page.query_selector('iframe[src*="wohnungshelden.de"]') if iframe_element: iframe_url = await iframe_element.get_attribute('src') logger.info(f"[DEGEWO] Found Wohnungshelden iframe: {iframe_url}") - # Navigate to the iframe URL directly in a new page for full access iframe_page = await self.context.new_page() try: await iframe_page.goto(iframe_url, wait_until="networkidle") await asyncio.sleep(2) logger.info("[DEGEWO] Loaded Wohnungshelden application page") - # TODO: Implement form-filling and submission logic here + + # Screenshot and HTML for debugging + screenshot_path = DATA_DIR / f"degewo_wohnungshelden_{listing['id']}.png" + await iframe_page.screenshot(path=str(screenshot_path), full_page=True) + logger.info(f"[DEGEWO] Saved Wohnungshelden screenshot to {screenshot_path}") + html_content = await iframe_page.content() + html_path = DATA_DIR / f"degewo_wohnungshelden_{listing['id']}.html" + with open(html_path, 'w', encoding='utf-8') as f: + f.write(html_content) + logger.info(f"[DEGEWO] Saved HTML to {html_path}") + + # Fill out Wohnungshelden form + form_filled = False + # Anrede (Salutation) + try: + salutation_dropdown = await iframe_page.query_selector('#salutation-dropdown, ng-select[id*="salutation"]') + if salutation_dropdown: + await salutation_dropdown.click() + await asyncio.sleep(0.5) + anrede_option = await iframe_page.query_selector(f'.ng-option:has-text("{os.environ.get("FORM_ANREDE", "Herr")}")') + if anrede_option: + await anrede_option.click() + logger.info(f"[DEGEWO] Selected Anrede: {os.environ.get('FORM_ANREDE', 'Herr')}") + form_filled = True + except Exception as e: + logger.warning(f"[DEGEWO] Could not set Anrede: {e}") + + # Vorname + try: + vorname_field = await iframe_page.query_selector('#firstName') + if vorname_field: + await vorname_field.fill(os.environ.get("FORM_VORNAME", "Max")) + logger.info(f"[DEGEWO] Filled Vorname: {os.environ.get('FORM_VORNAME', 'Max')}") + form_filled = True + except Exception as e: + logger.warning(f"[DEGEWO] Could not fill Vorname: {e}") + + # Nachname + try: + nachname_field = await iframe_page.query_selector('#lastName') + if nachname_field: + await nachname_field.fill(os.environ.get("FORM_NACHNAME", "Mustermann")) + logger.info(f"[DEGEWO] Filled Nachname: {os.environ.get('FORM_NACHNAME', 'Mustermann')}") + form_filled = True + except Exception as e: + logger.warning(f"[DEGEWO] Could not fill Nachname: {e}") + + # E-Mail + try: + email_field = await iframe_page.query_selector('#email') + if email_field: + await email_field.fill(os.environ.get("FORM_EMAIL", "test@example.com")) + logger.info(f"[DEGEWO] Filled E-Mail: {os.environ.get('FORM_EMAIL', 'test@example.com')}") + form_filled = True + except Exception as e: + logger.warning(f"[DEGEWO] Could not fill E-Mail: {e}") + + # Telefonnummer + try: + tel_field = await iframe_page.query_selector('input[id*="telefonnummer"]') + if tel_field: + await tel_field.fill(os.environ.get("FORM_PHONE", "0123456789")) + logger.info(f"[DEGEWO] Filled Telefon: {os.environ.get('FORM_PHONE', '0123456789')}") + form_filled = True + except Exception as e: + logger.warning(f"[DEGEWO] Could not fill Telefon: {e}") + + # Anzahl einziehende Personen + try: + personen_field = await iframe_page.query_selector('input[id*="numberPersonsTotal"]') + if personen_field: + await personen_field.fill(os.environ.get("FORM_PERSONS", "1")) + logger.info(f"[DEGEWO] Filled Anzahl Personen: {os.environ.get('FORM_PERSONS', '1')}") + form_filled = True + except Exception as e: + logger.warning(f"[DEGEWO] Could not fill Anzahl Personen: {e}") + + # "FΓΌr sich selbst" dropdown + try: + selbst_dropdown = await iframe_page.query_selector('ng-select[id*="fuer_wen"]') + if selbst_dropdown: + await selbst_dropdown.click() + await asyncio.sleep(0.5) + selbst_option = await iframe_page.query_selector('.ng-option:has-text("FΓΌr mich selbst"), .ng-option:has-text("selbst")') + if selbst_option: + await selbst_option.click() + logger.info("[DEGEWO] Selected: FΓΌr mich selbst") + form_filled = True + except Exception as e: + logger.warning(f"[DEGEWO] Could not set 'FΓΌr sich selbst': {e}") + + await asyncio.sleep(1) + + # Take screenshot after filling form + screenshot_path = DATA_DIR / f"degewo_form_filled_{listing['id']}.png" + await iframe_page.screenshot(path=str(screenshot_path), full_page=True) + logger.info(f"[DEGEWO] Saved filled form screenshot to {screenshot_path}") + + # Try to submit + try: + submit_selectors = [ + 'button[type="submit"]', + 'input[type="submit"]', + 'button:has-text("Absenden")', + 'button:has-text("Senden")', + 'button:has-text("Anfrage")', + 'button:has-text("Bewerben")', + 'button:has-text("Submit")', + '.btn-primary', + '.submit-btn', + ] + + submit_btn = None + for selector in submit_selectors: + submit_btn = await iframe_page.query_selector(selector) + if submit_btn and await submit_btn.is_visible(): + logger.info(f"[DEGEWO] Found submit button with selector: {selector}") + break + submit_btn = None + + if submit_btn: + await submit_btn.click() + logger.info("[DEGEWO] Clicked submit button") + await asyncio.sleep(3) + + # Take screenshot after submission + screenshot_path = DATA_DIR / f"degewo_submitted_{listing['id']}.png" + await iframe_page.screenshot(path=str(screenshot_path), full_page=True) + logger.info(f"[DEGEWO] Saved submission screenshot to {screenshot_path}") + + result["success"] = True + result["message"] = "Application submitted via Wohnungshelden" + else: + result["success"] = False + result["message"] = "Wohnungshelden form loaded but submit button not found" + logger.warning("[DEGEWO] Submit button not found in Wohnungshelden form") + except Exception as e: + result["success"] = False + result["message"] = f"Wohnungshelden submit error: {str(e)}" + logger.warning(f"[DEGEWO] Submit error: {e}") finally: await iframe_page.close() else: - # No iframe found - try the old approach (fallback for different page structure) logger.warning("[DEGEWO] Wohnungshelden iframe not found, trying direct form...") - # TODO: Implement fallback logic here + screenshot_path = DATA_DIR / f"degewo_noiframe_{listing['id']}.png" + await page.screenshot(path=str(screenshot_path), full_page=True) + html_content = await page.content() + html_path = DATA_DIR / "degewo_debug.html" + with open(html_path, 'w', encoding='utf-8') as f: + f.write(html_content) + result["success"] = False + result["message"] = "Wohnungshelden iframe not found on page" else: result["message"] = "No kontaktieren button found" logger.warning("[DEGEWO] Could not find kontaktieren button") screenshot_path = DATA_DIR / f"degewo_nobtn_{listing['id']}.png" await page.screenshot(path=str(screenshot_path), full_page=True) - await page.close() return result except Exception as e: diff --git a/handlers/wgcompany_notifier.py b/handlers/wgcompany_notifier.py index 26f3b72..2bc8fa0 100644 --- a/handlers/wgcompany_notifier.py +++ b/handlers/wgcompany_notifier.py @@ -38,6 +38,7 @@ class WGCompanyNotifier: logger.info("[WGCOMPANY] Browser initialized") async def fetch_listings(self): + await self.init_browser() listings = [] try: page = await self.context.new_page() diff --git a/requirements.txt b/requirements.txt index 5c094d8..39935e8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ requests>=2.31.0 +httpx>=0.24.0 playwright>=1.57.0 matplotlib>=3.8.0 pandas>=2.0.0 diff --git a/telegram_bot.py b/telegram_bot.py index 553efac..3879a02 100644 --- a/telegram_bot.py +++ b/telegram_bot.py @@ -1,4 +1,5 @@ + import os import logging import threading @@ -7,6 +8,7 @@ import requests import asyncio + # Configuration from environment TELEGRAM_BOT_TOKEN = os.environ.get("TELEGRAM_BOT_TOKEN", "") TELEGRAM_CHAT_ID = os.environ.get("TELEGRAM_CHAT_ID", "") @@ -16,9 +18,30 @@ logger = logging.getLogger(__name__) class TelegramBot: - """Handle Telegram commands for controlling the monitor""" - def _handle_reset_listings_command(self): + async def _handle_help_command(self): + """Send a help message with available commands.""" + help_text = ( + "Available commands:\n" + "/autopilot on|off - Enable/disable autopilot\n" + "/status - Show current status\n" + "/plot - Show weekly listing pattern plot\n" + "/errorrate - Show autopilot error rate plot\n" + "/retryfailed - Retry failed applications\n" + "/resetlistings - Reset listings file\n" + "/help - Show this help message" + ) + await self._send_message(help_text) + + async def _handle_unknown_command(self, text): + """Handle unknown commands and notify the user.""" + cmd = text.split()[0] if text else text + msg = ( + f"❓ Unknown command: {cmd}\n\nUse /help to see available commands." + ) + await self._send_message(msg) + + async def _handle_reset_listings_command(self): """Move listings.json to data/old/ with a timestamp, preserving statistics and application history.""" import shutil from datetime import datetime @@ -29,15 +52,20 @@ class TelegramBot: # Ensure old_dir exists os.makedirs(old_dir, exist_ok=True) timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - dest_path = os.path.join(old_dir, f"listings_{timestamp}.json") + dest_path = os.path.join( + old_dir, f"listings_{timestamp}.json" + ) shutil.move(listings_path, dest_path) - msg = f"πŸ—‘οΈ Listings reset:\nlistings.json moved to old/listings_{timestamp}.json." + msg = ( + f"πŸ—‘οΈ Listings reset:\nlistings.json moved to " + f"old/listings_{timestamp}.json." + ) else: msg = "ℹ️ No listings file found to move." - self._send_message(msg) + await self._send_message(msg) except Exception as e: logger.error(f"Error resetting listings: {e}") - self._send_message(f"❌ Error resetting listings: {str(e)}") + await self._send_message(f"❌ Error resetting listings: {str(e)}") def __init__(self, monitor, bot_token=None, chat_id=None, event_loop=None): self.monitor = monitor @@ -89,44 +117,31 @@ class TelegramBot: logger.debug(f"Ignoring message from unknown chat: {chat_id}") return logger.info(f"Received Telegram command: {text}") + loop = self.event_loop if text.startswith("/autopilot"): - self._handle_autopilot_command(text) + asyncio.run_coroutine_threadsafe(self._handle_autopilot_command(text), loop) elif text == "/status": - self._handle_status_command() + asyncio.run_coroutine_threadsafe(self._handle_status_command(), loop) elif text == "/help": - self._handle_help_command() + asyncio.run_coroutine_threadsafe(self._handle_help_command(), loop) elif text == "/plot": - self._handle_plot_command() + asyncio.run_coroutine_threadsafe(self._handle_plot_command(), loop) elif text == "/errorrate": - self._handle_error_rate_command() + asyncio.run_coroutine_threadsafe(self._handle_error_rate_command(), loop) elif text == "/retryfailed": - # Schedule coroutine on the main event loop for thread safety fut = asyncio.run_coroutine_threadsafe( self._handle_retry_failed_command(max_retries=TELEGRAM_MAX_RETRIES), - self.event_loop + loop ) - # Optionally, wait for result or handle exceptions try: fut.result() except Exception as e: logger.error(f"/retryfailed command failed: {e}") elif text == "/resetlistings": - self._handle_reset_listings_command() + asyncio.run_coroutine_threadsafe(self._handle_reset_listings_command(), loop) elif text.startswith("/"): - self._handle_unknown_command(text) - def _handle_reset_listings_command(self): - """Delete listings.json (not wgcompany_listings.json), but preserve statistics and application history.""" - try: - listings_path = os.path.join("data", "listings.json") - if os.path.exists(listings_path): - os.remove(listings_path) - msg = "πŸ—‘οΈ Listings reset:\nlistings.json deleted." - else: - msg = "ℹ️ No listings file found to delete." - self._send_message(msg) - except Exception as e: - logger.error(f"Error resetting listings: {e}") - self._send_message(f"❌ Error resetting listings: {str(e)}") + asyncio.run_coroutine_threadsafe(self._handle_unknown_command(text), loop) + async def _handle_retry_failed_command(self, max_retries: int = 3): """Retry all failed applications up to max_retries.""" # Ensure browser context is initialized @@ -137,11 +152,11 @@ class TelegramBot: if hasattr(self.app_handler, 'context') and hasattr(self.app_handler, 'handlers'): for handler in self.app_handler.handlers.values(): handler.context = self.app_handler.context - self._send_message(f"πŸ”„ Retrying failed applications (max retries: {max_retries})...") applications = self.app_handler.load_applications() failed = [app for app in applications.values() if not app.get("success") and app.get("retries", 0) < max_retries] + await self._send_message(f"πŸ”„ Retrying {len(failed)} failed applications (max retries: {max_retries})...") if not failed: - self._send_message("βœ… No failed applications to retry (or all reached max retries).") + await self._send_message("βœ… No failed applications to retry (or all reached max retries).") return results = {} details = [] @@ -170,26 +185,26 @@ class TelegramBot: summary = f"πŸ”„ Retried {len(results)} failed applications.\nβœ… Success: {n_success}\n❌ Still failed: {n_fail}" if details: summary += "\n\nDetails:\n" + "\n".join(details) - self._send_message(summary) + await self._send_message(summary) - def _handle_autopilot_command(self, text): + async def _handle_autopilot_command(self, text): logger.info(f"Processing autopilot command: {text}") parts = text.split() if len(parts) < 2: - self._send_message("Usage: /autopilot on|off") + await self._send_message("Usage: /autopilot on|off") return action = parts[1].lower() if action == "on": logger.info("Enabling autopilot mode") self.monitor.set_autopilot(True) - self._send_message("πŸ€– Autopilot ENABLED\n\nI will automatically apply to new listings!") + await self._send_message("πŸ€– Autopilot ENABLED\n\nI will automatically apply to new listings!") elif action == "off": self.monitor.set_autopilot(False) - self._send_message("πŸ›‘ Autopilot DISABLED\n\nI will only notify you of new listings.") + await self._send_message("πŸ›‘ Autopilot DISABLED\n\nI will only notify you of new listings.") else: - self._send_message("Usage: /autopilot on|off") + await self._send_message("Usage: /autopilot on|off") - def _handle_status_command(self): + async def _handle_status_command(self): state = self.app_handler.load_state() autopilot = state.get("autopilot", False) applications = self.app_handler.load_applications() @@ -203,89 +218,83 @@ class TelegramBot: status += "\n\nBy company:" for company, count in sorted(by_company.items()): status += f"\n β€’ {company}: {count}" - self._send_message(status) + await self._send_message(status) - def _handle_plot_command(self): + async def _handle_plot_command(self): logger.info("Generating listing times plot...") try: plot_path = self.app_handler._generate_weekly_plot() - self._send_photo(plot_path, "\U0001f4ca Weekly Listing Patterns\n\nThis shows when new listings typically appear throughout the week.") + await self._send_photo(plot_path, "\U0001f4ca Weekly Listing Patterns\n\nThis shows when new listings typically appear throughout the week.") except Exception as e: logger.error(f"Error generating plot: {e}") import traceback logger.error(traceback.format_exc()) - self._send_message(f"\u274c Error generating plot: {str(e)}") - cmd = text.split()[0] if text else text - self._send_message(f"❓ Unknown command: {cmd}\n\nUse /help to see available commands.") + await self._send_message(f"\u274c Error generating plot: {str(e)}") - def _handle_error_rate_command(self): + async def _handle_error_rate_command(self): logger.info("Generating autopilot errorrate plot...") try: plot_path, summary = self.app_handler._generate_error_rate_plot() caption = "πŸ“‰ Autopilot Success vs Failure\n\n" + summary - self._send_photo(plot_path, caption) + await self._send_photo(plot_path, caption) except Exception as e: logger.error(f"Error generating errorrate plot: {e}") import traceback logger.error(traceback.format_exc()) - self._send_message(f"❌ Error generating errorrate plot: {str(e)}") + await self._send_message(f"❌ Error generating errorrate plot: {str(e)}") - def _handle_plot_command(self): - logger.info("Generating listing times plot...") - try: - plot_path = self.app_handler._generate_weekly_plot() - if plot_path: - self._send_photo(plot_path, "πŸ“Š Weekly Listing Patterns\n\nThis shows when new listings typically appear throughout the week.") - else: - self._send_message("πŸ“Š Not enough data to generate plot yet. Keep monitoring!") - except Exception as e: - logger.error(f"Error generating plot: {e}") - import traceback - logger.error(traceback.format_exc()) - self._send_message(f"❌ Error generating plot: {str(e)}") - def _send_message(self, text): - """Send a text message to the configured Telegram chat, with detailed error logging.""" + async def _send_message(self, text): + """Send a text message to the configured Telegram chat, with detailed error logging (async).""" + import httpx + MAX_LENGTH = 4096 # Telegram message character limit if not self.bot_token or not self.chat_id: logger.warning("Telegram bot token or chat ID not configured, cannot send message") return url = f"https://api.telegram.org/bot{self.bot_token}/sendMessage" - payload = {"chat_id": self.chat_id, "text": text, "parse_mode": "HTML"} + # Split message into chunks if too long + messages = [] + if isinstance(text, str) and len(text) > MAX_LENGTH: + # Try to split on line breaks for readability + lines = text.split('\n') + chunk = "" + for line in lines: + if len(chunk) + len(line) + 1 > MAX_LENGTH: + messages.append(chunk) + chunk = line + else: + if chunk: + chunk += "\n" + chunk += line + if chunk: + messages.append(chunk) + else: + messages = [text] try: - response = requests.post(url, json=payload, timeout=10) - logger.info(f"[TELEGRAM] Sent message: status={response.status_code}, ok={response.ok}, response={response.text}") - if not response.ok: - logger.error(f"Failed to send Telegram message: {response.text}") + async with httpx.AsyncClient(timeout=10) as client: + for idx, msg in enumerate(messages): + payload = {"chat_id": self.chat_id, "text": msg, "parse_mode": "HTML"} + response = await client.post(url, json=payload) + logger.info(f"[TELEGRAM] Sent message part {idx+1}/{len(messages)}: status={response.status_code}, ok={response.is_success}") + if not response.is_success: + logger.error(f"Failed to send Telegram message: {response.text}") except Exception as e: logger.error(f"Error while sending Telegram message: {e}") - import traceback - logger.error(traceback.format_exc()) - def _send_photo(self, photo_path, caption): - """Send a photo to the configured Telegram chat.""" + async def _send_photo(self, photo_path, caption): + """Send a photo to the configured Telegram chat (async).""" + import httpx if not self.bot_token or not self.chat_id: logger.warning("Telegram bot token or chat ID not configured, cannot send photo") return url = f"https://api.telegram.org/bot{self.bot_token}/sendPhoto" - with open(photo_path, "rb") as photo: - payload = {"chat_id": self.chat_id, "caption": caption, "parse_mode": "HTML"} - files = {"photo": photo} - try: - response = requests.post(url, data=payload, files=files, timeout=10) - if not response.ok: - logger.error(f"Failed to send Telegram photo: {response.text}") - except Exception as e: - logger.error(f"Error while sending Telegram photo: {e}") - - def _generate_error_rate_plot(self): - """Generate and send a plot showing success vs failure ratio for autopilot applications.""" - logger.info("Generating autopilot errorrate plot...") try: - plot_path, summary = self.app_handler._generate_error_rate_plot() - if plot_path: - self._send_photo(plot_path, caption=summary) - else: - self._send_message("No data available to generate the error rate plot.") + with open(photo_path, "rb") as photo: + files = {"photo": (photo_path, photo, "image/jpeg")} + data = {"chat_id": self.chat_id, "caption": caption, "parse_mode": "HTML"} + async with httpx.AsyncClient(timeout=10) as client: + response = await client.post(url, data=data, files=files) + if not response.is_success: + logger.error(f"Failed to send Telegram photo: {response.text}") except Exception as e: - logger.error(f"Error generating errorrate plot: {e}") - self._send_message(f"❌ Error generating errorrate plot: {str(e)}") \ No newline at end of file + logger.error(f"Error while sending Telegram photo: {e}") diff --git a/tests/test_application_handler.py b/tests/test_application_handler.py index 0671c35..569ee71 100644 --- a/tests/test_application_handler.py +++ b/tests/test_application_handler.py @@ -16,12 +16,18 @@ def temp_applications_file(tmp_path): @pytest.fixture def application_handler(temp_applications_file, monkeypatch): """Fixture to create an ApplicationHandler instance with a temporary applications file.""" + class DummyContext: pass + class DummyStateManager: pass monkeypatch.setattr("application_handler.APPLICATIONS_FILE", temp_applications_file) - return ApplicationHandler(browser_context=None, state_manager=None) + return ApplicationHandler(browser_context=DummyContext(), state_manager=DummyStateManager(), applications_file=temp_applications_file) +import types +class DummyContext: pass +class DummyStateManager: pass + def test_detect_company_domains(): - handler = ApplicationHandler(browser_context=None, state_manager=None) + handler = ApplicationHandler(browser_context=DummyContext(), state_manager=DummyStateManager()) assert handler._detect_company('https://howoge.de/abc') == 'howoge' assert handler._detect_company('https://www.howoge.de/abc') == 'howoge' assert handler._detect_company('https://portal.gewobag.de/') == 'gewobag' @@ -32,7 +38,7 @@ def test_detect_company_domains(): assert handler._detect_company('https://wbm.de/') == 'wbm' def test_detect_company_path_fallback(): - handler = ApplicationHandler(browser_context=None, state_manager=None) + handler = ApplicationHandler(browser_context=DummyContext(), state_manager=DummyStateManager()) assert handler._detect_company('https://example.com/howoge/abc') == 'howoge' assert handler._detect_company('https://foo.bar/gewobag') == 'gewobag' assert handler._detect_company('https://foo.bar/degewo') == 'degewo' @@ -41,7 +47,7 @@ def test_detect_company_path_fallback(): assert handler._detect_company('https://foo.bar/wbm') == 'wbm' def test_detect_company_unknown(): - handler = ApplicationHandler(browser_context=None, state_manager=None) + handler = ApplicationHandler(browser_context=DummyContext(), state_manager=DummyStateManager()) assert handler._detect_company('https://example.com/') == 'unknown' assert handler._detect_company('') == 'unknown' assert handler._detect_company(None) == 'unknown' diff --git a/tests/test_company_detection.py b/tests/test_company_detection.py index 9c7d012..e591968 100644 --- a/tests/test_company_detection.py +++ b/tests/test_company_detection.py @@ -13,7 +13,8 @@ class DummyStateManager: def make_handler(): # context is not used for _detect_company - return ApplicationHandler(browser_context=None, state_manager=DummyStateManager()) + class DummyContext: pass + return ApplicationHandler(browser_context=DummyContext(), state_manager=DummyStateManager()) def test_detect_company_domains(): handler = make_handler() diff --git a/tests/test_error_rate_plot.py b/tests/test_error_rate_plot.py index 342ae73..15ffa1b 100644 --- a/tests/test_error_rate_plot.py +++ b/tests/test_error_rate_plot.py @@ -24,17 +24,23 @@ class DummyStateManager: def is_autopilot_enabled(self): return False + @patch("matplotlib.pyplot.savefig") def test_generate_error_rate_plot_no_data(mock_savefig, temp_applications_file): - handler = ApplicationHandler(None, DummyStateManager(), applications_file=temp_applications_file) + class DummyContext: pass + handler = ApplicationHandler(DummyContext(), DummyStateManager(), applications_file=temp_applications_file) plot_path, summary = handler._generate_error_rate_plot() assert plot_path is None or plot_path == "" assert summary == "" -@patch("matplotlib.pyplot.savefig") + +from unittest.mock import patch + +@patch("matplotlib.figure.Figure.savefig") def test_generate_error_rate_plot_with_data(mock_savefig, temp_applications_file): - handler = ApplicationHandler(None, DummyStateManager(), applications_file=temp_applications_file) + class DummyContext: pass + handler = ApplicationHandler(DummyContext(), DummyStateManager(), applications_file=temp_applications_file) # Write valid data to the temp applications file temp_applications_file.write_text(''' { @@ -48,4 +54,4 @@ def test_generate_error_rate_plot_with_data(mock_savefig, temp_applications_file assert "Successes" in summary assert "Failures" in summary assert "Overall error rate" in summary - mock_savefig.assert_called_once() \ No newline at end of file + mock_savefig.assert_called() \ No newline at end of file diff --git a/tests/test_handlers.py b/tests/test_handlers.py index bc790db..f2aa38d 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -1,3 +1,6 @@ +import sys +from pathlib import Path +sys.path.append(str(Path(__file__).parent.parent)) import pytest from handlers.howoge_handler import HowogeHandler from handlers.gewobag_handler import GewobagHandler @@ -16,7 +19,7 @@ class MockBaseHandler(BaseHandler): async def test_howoge_handler(): context = AsyncMock() handler = HowogeHandler(context) - listing = {"link": "https://www.howoge.de/example"} + listing = {"link": "https://www.howoge.de/example", "id": "testid"} result = {"success": False} await handler.apply(listing, result) assert "success" in result