From 54f0c064cdc36e0cd62fae8fe48e40a8e7d79c70 Mon Sep 17 00:00:00 2001 From: Mannu Date: Thu, 30 Apr 2026 22:41:10 +0530 Subject: [PATCH] homepage: config writer + seed scripts for miniflux and changedetection --- seed_changedetection.py | 153 ++++++++++++++++++++++++++++++++ seed_miniflux.py | 182 +++++++++++++++++++++++++++++++++++++++ write_homepage_config.py | 124 ++++++++++++++++++++++++++ 3 files changed, 459 insertions(+) create mode 100644 seed_changedetection.py create mode 100644 seed_miniflux.py create mode 100644 write_homepage_config.py diff --git a/seed_changedetection.py b/seed_changedetection.py new file mode 100644 index 0000000..a6ee36a --- /dev/null +++ b/seed_changedetection.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +""" +seed_changedetection.py -- adds Indian RE regulator pages to ChangeDetection. +These are pages with NO RSS feed -- we watch for any content change. + +Notifications route via Apprise -> Telegram. +Run: python3 seed_changedetection.py +""" + +import requests +import sys + +# ── Config ──────────────────────────────────────────────────────────────────── +CD_URL = "https://watch.manohargupta.com" +CD_USER = "manohar" +CD_PASS = "your_changedetection_password" # the password you set with htpasswd + +# Apprise notification URL -- change alerts go here -> Telegram +APPRISE_NOTIFY_URL = "https://manohar:your_apprise_password@notify.manohargupta.com/notify/apprise" + +# ── Watch definitions ───────────────────────────────────────────────────────── +# Format: (title, url, check_interval_seconds, tag) +# Interval guide: 3600=1hr, 7200=2hr, 14400=4hr, 43200=12hr, 86400=24hr +WATCHES = [ + # Regulators -- check every 2 hours + ("CERC Orders", + "https://cercind.gov.in/orders.html", + 7200, "regulatory"), + + ("CERC Regulations", + "https://cercind.gov.in/regulations.html", + 7200, "regulatory"), + + ("MNRE Notifications", + "https://mnre.gov.in/notification", + 14400, "regulatory"), + + ("MNRE Tenders", + "https://mnre.gov.in/tender", + 14400, "regulatory"), + + ("SECI Tenders", + "https://seci.co.in/tenders.php", + 14400, "regulatory"), + + ("SECI Results/Awards", + "https://seci.co.in/tenders-awards.php", + 14400, "regulatory"), + + # Policy -- check every 12 hours + ("MoP Press Releases", + "https://powermin.gov.in/en/content/press-release", + 43200, "policy"), + + ("PPAC Petroleum Data", + "https://ppac.gov.in/content/212_1_PricesandTaxes.aspx", + 86400, "policy"), +] + + +# ── API helpers ─────────────────────────────────────────────────────────────── + +def api(method, path, **kwargs): + resp = requests.request( + method, + f"{CD_URL}/{path}", + auth=(CD_USER, CD_PASS), + headers={"Content-Type": "application/json"}, + timeout=15, + verify=True, + **kwargs + ) + return resp + + +def get_existing_urls(): + """Return set of URLs already being watched.""" + resp = api("GET", "api/v1/watch") + if resp.status_code != 200: + print(f"[ERROR] Could not fetch watches: {resp.status_code}") + return set() + watches = resp.json() + return {w.get("url", "") for w in watches.values()} + + +def add_watch(title, url, interval, tag): + """Add a watch. Returns (success, message).""" + payload = { + "url": url, + "title": title, + "time_between_check": { + "seconds": 0, + "minutes": 0, + "hours": interval // 3600, + "days": 0, + "weeks": 0, + }, + "tag": tag, + # Route all change notifications through Apprise + "notification_urls": [APPRISE_NOTIFY_URL], + # Use the Browserless sidecar for JS-rendered pages + "fetch_backend": "playwright_chromium", + } + resp = api("POST", "api/v1/watch", json=payload) + if resp.status_code in (200, 201): + watch_id = resp.json().get("uuid", "?") + return True, f"added (id={watch_id})" + else: + try: + err = resp.json().get("error", resp.text[:100]) + except Exception: + err = resp.text[:100] + return False, f"FAILED: {err}" + + +# ── Main ────────────────────────────────────────────────────────────────────── + +def main(): + print(f"Connecting to {CD_URL}...") + resp = api("GET", "api/v1/watch") + if resp.status_code == 401: + print("Auth failed -- check CD_PASS in this script") + sys.exit(1) + elif resp.status_code != 200: + print(f"Connection failed: {resp.status_code}") + sys.exit(1) + + existing_urls = get_existing_urls() + print(f"Found {len(existing_urls)} existing watches\n") + + results = {"ok": [], "skip": [], "fail": []} + + for title, url, interval, tag in WATCHES: + if url in existing_urls: + print(f" ⏭️ [{tag}] {title}: already watching") + results["skip"].append(title) + continue + ok, msg = add_watch(title, url, interval, tag) + status = "✅" if ok else "❌" + print(f" {status} [{tag}] {title}: {msg}") + if ok: + results["ok"].append(title) + else: + results["fail"].append((title, msg)) + + print(f"\n{'='*50}") + print(f"Done. {len(results['ok'])} added, {len(results['skip'])} skipped, {len(results['fail'])} failed.") + print("\nChangeDetection will run first checks within the next refresh cycle.") + print("To force immediate check: Dashboard -> select all -> Recheck") + + +if __name__ == "__main__": + main() diff --git a/seed_miniflux.py b/seed_miniflux.py new file mode 100644 index 0000000..4f3d103 --- /dev/null +++ b/seed_miniflux.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +""" +seed_miniflux.py -- populates Miniflux with Indian RE + finance RSS feeds. +Uses Miniflux API with basic auth. +Run: python3 seed_miniflux.py + +Feed categories: + - Regulatory & Policy + - Industry & Analysis + - Markets + - Macro & Economy +""" + +import requests +import json +import sys + +# ── Config ──────────────────────────────────────────────────────────────────── +MINIFLUX_URL = "https://feeds.manohargupta.com" +MINIFLUX_USER = "manohar" +MINIFLUX_PASS = "QwEzVucn8Mm+ONROWPDGpA==" + +# ── Feed definitions ────────────────────────────────────────────────────────── +# Format: (title, url, category, crawler) +# crawler=True for JS-heavy sites that need full-page fetch +FEEDS = [ + + # ── Regulatory & Policy ─────────────────────────────────────────────────── + ("PIB - New & Renewable Energy", + "https://pib.gov.in/RssMain.aspx?ModId=6&Lang=1&Regid=3", + "Regulatory & Policy", False), + + ("PIB - Power Ministry", + "https://pib.gov.in/RssMain.aspx?ModId=7&Lang=1&Regid=3", + "Regulatory & Policy", False), + + # ── Industry & Analysis ─────────────────────────────────────────────────── + ("Mercom India", + "https://mercomindia.com/feed/", + "Industry & Analysis", False), + + ("IEEFA", + "https://ieefa.org/feed/", + "Industry & Analysis", False), + + ("PV Magazine India", + "https://www.pv-magazine-india.com/feed/", + "Industry & Analysis", False), + + ("CleanTechnica", + "https://cleantechnica.com/feed/", + "Industry & Analysis", False), + + ("Bridge to India", + "https://bridgetoindia.com/feed/", + "Industry & Analysis", False), + + # ── Markets ─────────────────────────────────────────────────────────────── + ("Economic Times - Energy", + "https://energy.economictimes.indiatimes.com/rss/topstories", + "Markets", False), + + ("Moneycontrol - Latest News", + "https://www.moneycontrol.com/rss/latestnews.xml", + "Markets", False), + + ("Livemint - Companies", + "https://www.livemint.com/rss/companies", + "Markets", False), + + # ── Macro & Economy ─────────────────────────────────────────────────────── + ("RBI Press Releases", + "https://rbi.org.in/Scripts/RSS.aspx?Id=15", + "Macro & Economy", False), + + ("IMF - South Asia", + "https://www.imf.org/en/News/rss?language=eng", + "Macro & Economy", False), + + ("CEEW", + "https://www.ceew.in/rss.xml", + "Macro & Economy", False), +] + + +# ── API helpers ─────────────────────────────────────────────────────────────── + +def api(method, path, **kwargs): + resp = requests.request( + method, + f"{MINIFLUX_URL}/v1{path}", + auth=(MINIFLUX_USER, MINIFLUX_PASS), + headers={"Content-Type": "application/json"}, + timeout=15, + **kwargs + ) + return resp + + +def get_or_create_category(name, existing): + """Return category ID, creating it if it doesn't exist.""" + for cat in existing: + if cat["title"] == name: + return cat["id"] + resp = api("POST", "/categories", json={"title": name}) + if resp.status_code == 201: + cat_id = resp.json()["id"] + print(f" [CAT] Created category: {name} (id={cat_id})") + return cat_id + else: + print(f" [ERROR] Could not create category {name}: {resp.text}") + return None + + +def add_feed(title, url, category_id, crawler): + """Add a feed. Returns (success, message).""" + payload = { + "feed_url": url, + "category_id": category_id, + "crawler": crawler, # fetch full page for JS-heavy sites + "user_agent": "Mozilla/5.0 (compatible; Miniflux)", + } + resp = api("POST", "/feeds", json=payload) + if resp.status_code == 201: + feed_id = resp.json()["feed_id"] + return True, f"added (id={feed_id})" + elif resp.status_code == 409: + return True, "already exists" + else: + try: + err = resp.json().get("error_message", resp.text) + except Exception: + err = resp.text + return False, f"FAILED: {err}" + + +# ── Main ────────────────────────────────────────────────────────────────────── + +def main(): + print(f"Connecting to {MINIFLUX_URL}...") + me = api("GET", "/me") + if me.status_code != 200: + print(f"Auth failed: {me.status_code} {me.text}") + sys.exit(1) + print(f"Authenticated as: {me.json()['username']}\n") + + # Fetch existing categories + cats_resp = api("GET", "/categories") + existing_cats = cats_resp.json() if cats_resp.status_code == 200 else [] + + results = {"ok": [], "fail": []} + + for title, url, category_name, crawler in FEEDS: + cat_id = get_or_create_category(category_name, existing_cats) + # Refresh category list after creation + cats_resp = api("GET", "/categories") + existing_cats = cats_resp.json() if cats_resp.status_code == 200 else existing_cats + + if not cat_id: + results["fail"].append((title, "no category")) + continue + + ok, msg = add_feed(title, url, cat_id, crawler) + status = "✅" if ok else "❌" + print(f" {status} [{category_name}] {title}: {msg}") + if ok: + results["ok"].append(title) + else: + results["fail"].append((title, msg)) + + print(f"\n{'='*50}") + print(f"Done. {len(results['ok'])} added, {len(results['fail'])} failed.") + if results["fail"]: + print("\nFailed feeds (check URL or site availability):") + for title, reason in results["fail"]: + print(f" - {title}: {reason}") + print("\nMiniflux will fetch all feeds in the next refresh cycle (~1 hour).") + print("Force a refresh: Feeds → select all → Refresh") + + +if __name__ == "__main__": + main() diff --git a/write_homepage_config.py b/write_homepage_config.py new file mode 100644 index 0000000..fc3c556 --- /dev/null +++ b/write_homepage_config.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Writes Homepage config files directly into the volume.""" + +import os + +CONFIG_DIR = "/var/lib/docker/volumes/selfhosted-home-diqpg6_homepage_config/_data" + +SETTINGS = """title: Manohar's Hub +theme: dark +color: slate +headerStyle: boxed +layout: + Infrastructure: + style: row + columns: 4 + Tools: + style: row + columns: 4 + Dev: + style: row + columns: 4 + Finance & Intel: + style: row + columns: 4 +""" + +SERVICES = """- Infrastructure: + - Dokploy: + href: https://dokploy.manohargupta.com + description: Docker orchestration + icon: docker.png + - Uptime Kuma: + href: https://status.manohargupta.com + description: Service monitoring + icon: uptime-kuma.png + - Umami: + href: https://analytics.manohargupta.com + description: Web analytics + icon: umami.png + - Traefik: + href: https://dokploy.manohargupta.com + description: Reverse proxy + TLS + icon: traefik.png + +- Tools: + - n8n: + href: https://automate.manohargupta.com + description: Workflow automation + icon: n8n.png + - Paperless: + href: https://docs.manohargupta.com + description: Document OCR + search + icon: paperless-ngx.png + - Miniflux: + href: https://feeds.manohargupta.com + description: RSS reader + icon: miniflux.png + - ChangeDetection: + href: https://watch.manohargupta.com + description: Page change monitor + icon: changedetection-io.png + +- Dev: + - Forgejo: + href: https://git.manohargupta.com + description: Self-hosted Git + icon: gitea.png + - Code Server: + href: https://code.manohargupta.com + description: Browser IDE + icon: code-server.png + - Apprise: + href: https://notify.manohargupta.com + description: Notification hub + icon: apprise.png + +- Finance & Intel: + - Tiger Agent: + href: https://agent.manohargupta.com + description: AI orchestration + icon: https://cdn.jsdelivr.net/gh/walkxcode/dashboard-icons/png/heimdall.png +""" + +BOOKMARKS = """- Quick Links: + - Namecheap DNS: + - href: https://ap.www.namecheap.com/domains/domaincontrolpanel/manohargupta.com/advancedns + - Hetzner Console: + - href: https://console.hetzner.cloud + - OpenRouter: + - href: https://openrouter.ai/activity + - BotFather: + - href: https://t.me/BotFather +""" + +WIDGETS = """- greeting: + text_size: xl + text: "Good morning, Manohar" + +- datetime: + text_size: xl + format: + timeStyle: short + dateStyle: short + hourCycle: h23 + +- search: + provider: google + target: _blank +""" + +files = { + "settings.yaml": SETTINGS, + "services.yaml": SERVICES, + "bookmarks.yaml": BOOKMARKS, + "widgets.yaml": WIDGETS, +} + +for filename, content in files.items(): + path = os.path.join(CONFIG_DIR, filename) + with open(path, "w") as f: + f.write(content) + print(f"Written: {path}") + +print("All config files written.")