homepage: config writer + seed scripts for miniflux and changedetection

seed_changedetection.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+"""
+seed_changedetection.py -- adds Indian RE regulator pages to ChangeDetection.
+These are pages with NO RSS feed -- we watch for any content change.
+
+Notifications route via Apprise -> Telegram.
+Run: python3 seed_changedetection.py
+"""
+
+import requests
+import sys
+
+# ── Config ────────────────────────────────────────────────────────────────────
+CD_URL      = "https://watch.manohargupta.com"
+CD_USER     = "manohar"
+CD_PASS     = "your_changedetection_password"   # the password you set with htpasswd
+
+# Apprise notification URL -- change alerts go here -> Telegram
+APPRISE_NOTIFY_URL = "https://manohar:your_apprise_password@notify.manohargupta.com/notify/apprise"
+
+# ── Watch definitions ─────────────────────────────────────────────────────────
+# Format: (title, url, check_interval_seconds, tag)
+# Interval guide: 3600=1hr, 7200=2hr, 14400=4hr, 43200=12hr, 86400=24hr
+WATCHES = [
+    # Regulators -- check every 2 hours
+    ("CERC Orders",
+     "https://cercind.gov.in/orders.html",
+     7200, "regulatory"),
+
+    ("CERC Regulations",
+     "https://cercind.gov.in/regulations.html",
+     7200, "regulatory"),
+
+    ("MNRE Notifications",
+     "https://mnre.gov.in/notification",
+     14400, "regulatory"),
+
+    ("MNRE Tenders",
+     "https://mnre.gov.in/tender",
+     14400, "regulatory"),
+
+    ("SECI Tenders",
+     "https://seci.co.in/tenders.php",
+     14400, "regulatory"),
+
+    ("SECI Results/Awards",
+     "https://seci.co.in/tenders-awards.php",
+     14400, "regulatory"),
+
+    # Policy -- check every 12 hours
+    ("MoP Press Releases",
+     "https://powermin.gov.in/en/content/press-release",
+     43200, "policy"),
+
+    ("PPAC Petroleum Data",
+     "https://ppac.gov.in/content/212_1_PricesandTaxes.aspx",
+     86400, "policy"),
+]
+
+
+# ── API helpers ───────────────────────────────────────────────────────────────
+
+def api(method, path, **kwargs):
+    resp = requests.request(
+        method,
+        f"{CD_URL}/{path}",
+        auth=(CD_USER, CD_PASS),
+        headers={"Content-Type": "application/json"},
+        timeout=15,
+        verify=True,
+        **kwargs
+    )
+    return resp
+
+
+def get_existing_urls():
+    """Return set of URLs already being watched."""
+    resp = api("GET", "api/v1/watch")
+    if resp.status_code != 200:
+        print(f"[ERROR] Could not fetch watches: {resp.status_code}")
+        return set()
+    watches = resp.json()
+    return {w.get("url", "") for w in watches.values()}
+
+
+def add_watch(title, url, interval, tag):
+    """Add a watch. Returns (success, message)."""
+    payload = {
+        "url": url,
+        "title": title,
+        "time_between_check": {
+            "seconds": 0,
+            "minutes": 0,
+            "hours": interval // 3600,
+            "days": 0,
+            "weeks": 0,
+        },
+        "tag": tag,
+        # Route all change notifications through Apprise
+        "notification_urls": [APPRISE_NOTIFY_URL],
+        # Use the Browserless sidecar for JS-rendered pages
+        "fetch_backend": "playwright_chromium",
+    }
+    resp = api("POST", "api/v1/watch", json=payload)
+    if resp.status_code in (200, 201):
+        watch_id = resp.json().get("uuid", "?")
+        return True, f"added (id={watch_id})"
+    else:
+        try:
+            err = resp.json().get("error", resp.text[:100])
+        except Exception:
+            err = resp.text[:100]
+        return False, f"FAILED: {err}"
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+def main():
+    print(f"Connecting to {CD_URL}...")
+    resp = api("GET", "api/v1/watch")
+    if resp.status_code == 401:
+        print("Auth failed -- check CD_PASS in this script")
+        sys.exit(1)
+    elif resp.status_code != 200:
+        print(f"Connection failed: {resp.status_code}")
+        sys.exit(1)
+
+    existing_urls = get_existing_urls()
+    print(f"Found {len(existing_urls)} existing watches\n")
+
+    results = {"ok": [], "skip": [], "fail": []}
+
+    for title, url, interval, tag in WATCHES:
+        if url in existing_urls:
+            print(f"  ⏭️  [{tag}] {title}: already watching")
+            results["skip"].append(title)
+            continue
+        ok, msg = add_watch(title, url, interval, tag)
+        status = "✅" if ok else "❌"
+        print(f"  {status} [{tag}] {title}: {msg}")
+        if ok:
+            results["ok"].append(title)
+        else:
+            results["fail"].append((title, msg))
+
+    print(f"\n{'='*50}")
+    print(f"Done. {len(results['ok'])} added, {len(results['skip'])} skipped, {len(results['fail'])} failed.")
+    print("\nChangeDetection will run first checks within the next refresh cycle.")
+    print("To force immediate check: Dashboard -> select all -> Recheck")
+
+
+if __name__ == "__main__":
+    main()

seed_miniflux.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+"""
+seed_miniflux.py -- populates Miniflux with Indian RE + finance RSS feeds.
+Uses Miniflux API with basic auth.
+Run: python3 seed_miniflux.py
+
+Feed categories:
+  - Regulatory & Policy
+  - Industry & Analysis
+  - Markets
+  - Macro & Economy
+"""
+
+import requests
+import json
+import sys
+
+# ── Config ────────────────────────────────────────────────────────────────────
+MINIFLUX_URL  = "https://feeds.manohargupta.com"
+MINIFLUX_USER = "manohar"
+MINIFLUX_PASS = "QwEzVucn8Mm+ONROWPDGpA=="
+
+# ── Feed definitions ──────────────────────────────────────────────────────────
+# Format: (title, url, category, crawler)
+# crawler=True for JS-heavy sites that need full-page fetch
+FEEDS = [
+
+    # ── Regulatory & Policy ───────────────────────────────────────────────────
+    ("PIB - New & Renewable Energy",
+     "https://pib.gov.in/RssMain.aspx?ModId=6&Lang=1&Regid=3",
+     "Regulatory & Policy", False),
+
+    ("PIB - Power Ministry",
+     "https://pib.gov.in/RssMain.aspx?ModId=7&Lang=1&Regid=3",
+     "Regulatory & Policy", False),
+
+    # ── Industry & Analysis ───────────────────────────────────────────────────
+    ("Mercom India",
+     "https://mercomindia.com/feed/",
+     "Industry & Analysis", False),
+
+    ("IEEFA",
+     "https://ieefa.org/feed/",
+     "Industry & Analysis", False),
+
+    ("PV Magazine India",
+     "https://www.pv-magazine-india.com/feed/",
+     "Industry & Analysis", False),
+
+    ("CleanTechnica",
+     "https://cleantechnica.com/feed/",
+     "Industry & Analysis", False),
+
+    ("Bridge to India",
+     "https://bridgetoindia.com/feed/",
+     "Industry & Analysis", False),
+
+    # ── Markets ───────────────────────────────────────────────────────────────
+    ("Economic Times - Energy",
+     "https://energy.economictimes.indiatimes.com/rss/topstories",
+     "Markets", False),
+
+    ("Moneycontrol - Latest News",
+     "https://www.moneycontrol.com/rss/latestnews.xml",
+     "Markets", False),
+
+    ("Livemint - Companies",
+     "https://www.livemint.com/rss/companies",
+     "Markets", False),
+
+    # ── Macro & Economy ───────────────────────────────────────────────────────
+    ("RBI Press Releases",
+     "https://rbi.org.in/Scripts/RSS.aspx?Id=15",
+     "Macro & Economy", False),
+
+    ("IMF - South Asia",
+     "https://www.imf.org/en/News/rss?language=eng",
+     "Macro & Economy", False),
+
+    ("CEEW",
+     "https://www.ceew.in/rss.xml",
+     "Macro & Economy", False),
+]
+
+
+# ── API helpers ───────────────────────────────────────────────────────────────
+
+def api(method, path, **kwargs):
+    resp = requests.request(
+        method,
+        f"{MINIFLUX_URL}/v1{path}",
+        auth=(MINIFLUX_USER, MINIFLUX_PASS),
+        headers={"Content-Type": "application/json"},
+        timeout=15,
+        **kwargs
+    )
+    return resp
+
+
+def get_or_create_category(name, existing):
+    """Return category ID, creating it if it doesn't exist."""
+    for cat in existing:
+        if cat["title"] == name:
+            return cat["id"]
+    resp = api("POST", "/categories", json={"title": name})
+    if resp.status_code == 201:
+        cat_id = resp.json()["id"]
+        print(f"  [CAT] Created category: {name} (id={cat_id})")
+        return cat_id
+    else:
+        print(f"  [ERROR] Could not create category {name}: {resp.text}")
+        return None
+
+
+def add_feed(title, url, category_id, crawler):
+    """Add a feed. Returns (success, message)."""
+    payload = {
+        "feed_url": url,
+        "category_id": category_id,
+        "crawler": crawler,           # fetch full page for JS-heavy sites
+        "user_agent": "Mozilla/5.0 (compatible; Miniflux)",
+    }
+    resp = api("POST", "/feeds", json=payload)
+    if resp.status_code == 201:
+        feed_id = resp.json()["feed_id"]
+        return True, f"added (id={feed_id})"
+    elif resp.status_code == 409:
+        return True, "already exists"
+    else:
+        try:
+            err = resp.json().get("error_message", resp.text)
+        except Exception:
+            err = resp.text
+        return False, f"FAILED: {err}"
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+def main():
+    print(f"Connecting to {MINIFLUX_URL}...")
+    me = api("GET", "/me")
+    if me.status_code != 200:
+        print(f"Auth failed: {me.status_code} {me.text}")
+        sys.exit(1)
+    print(f"Authenticated as: {me.json()['username']}\n")
+
+    # Fetch existing categories
+    cats_resp = api("GET", "/categories")
+    existing_cats = cats_resp.json() if cats_resp.status_code == 200 else []
+
+    results = {"ok": [], "fail": []}
+
+    for title, url, category_name, crawler in FEEDS:
+        cat_id = get_or_create_category(category_name, existing_cats)
+        # Refresh category list after creation
+        cats_resp = api("GET", "/categories")
+        existing_cats = cats_resp.json() if cats_resp.status_code == 200 else existing_cats
+
+        if not cat_id:
+            results["fail"].append((title, "no category"))
+            continue
+
+        ok, msg = add_feed(title, url, cat_id, crawler)
+        status = "✅" if ok else "❌"
+        print(f"  {status} [{category_name}] {title}: {msg}")
+        if ok:
+            results["ok"].append(title)
+        else:
+            results["fail"].append((title, msg))
+
+    print(f"\n{'='*50}")
+    print(f"Done. {len(results['ok'])} added, {len(results['fail'])} failed.")
+    if results["fail"]:
+        print("\nFailed feeds (check URL or site availability):")
+        for title, reason in results["fail"]:
+            print(f"  - {title}: {reason}")
+    print("\nMiniflux will fetch all feeds in the next refresh cycle (~1 hour).")
+    print("Force a refresh: Feeds → select all → Refresh")
+
+
+if __name__ == "__main__":
+    main()

write_homepage_config.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+"""Writes Homepage config files directly into the volume."""
+
+import os
+
+CONFIG_DIR = "/var/lib/docker/volumes/selfhosted-home-diqpg6_homepage_config/_data"
+
+SETTINGS = """title: Manohar's Hub
+theme: dark
+color: slate
+headerStyle: boxed
+layout:
+  Infrastructure:
+    style: row
+    columns: 4
+  Tools:
+    style: row
+    columns: 4
+  Dev:
+    style: row
+    columns: 4
+  Finance & Intel:
+    style: row
+    columns: 4
+"""
+
+SERVICES = """- Infrastructure:
+    - Dokploy:
+        href: https://dokploy.manohargupta.com
+        description: Docker orchestration
+        icon: docker.png
+    - Uptime Kuma:
+        href: https://status.manohargupta.com
+        description: Service monitoring
+        icon: uptime-kuma.png
+    - Umami:
+        href: https://analytics.manohargupta.com
+        description: Web analytics
+        icon: umami.png
+    - Traefik:
+        href: https://dokploy.manohargupta.com
+        description: Reverse proxy + TLS
+        icon: traefik.png
+
+- Tools:
+    - n8n:
+        href: https://automate.manohargupta.com
+        description: Workflow automation
+        icon: n8n.png
+    - Paperless:
+        href: https://docs.manohargupta.com
+        description: Document OCR + search
+        icon: paperless-ngx.png
+    - Miniflux:
+        href: https://feeds.manohargupta.com
+        description: RSS reader
+        icon: miniflux.png
+    - ChangeDetection:
+        href: https://watch.manohargupta.com
+        description: Page change monitor
+        icon: changedetection-io.png
+
+- Dev:
+    - Forgejo:
+        href: https://git.manohargupta.com
+        description: Self-hosted Git
+        icon: gitea.png
+    - Code Server:
+        href: https://code.manohargupta.com
+        description: Browser IDE
+        icon: code-server.png
+    - Apprise:
+        href: https://notify.manohargupta.com
+        description: Notification hub
+        icon: apprise.png
+
+- Finance & Intel:
+    - Tiger Agent:
+        href: https://agent.manohargupta.com
+        description: AI orchestration
+        icon: https://cdn.jsdelivr.net/gh/walkxcode/dashboard-icons/png/heimdall.png
+"""
+
+BOOKMARKS = """- Quick Links:
+    - Namecheap DNS:
+        - href: https://ap.www.namecheap.com/domains/domaincontrolpanel/manohargupta.com/advancedns
+    - Hetzner Console:
+        - href: https://console.hetzner.cloud
+    - OpenRouter:
+        - href: https://openrouter.ai/activity
+    - BotFather:
+        - href: https://t.me/BotFather
+"""
+
+WIDGETS = """- greeting:
+    text_size: xl
+    text: "Good morning, Manohar"
+
+- datetime:
+    text_size: xl
+    format:
+      timeStyle: short
+      dateStyle: short
+      hourCycle: h23
+
+- search:
+    provider: google
+    target: _blank
+"""
+
+files = {
+    "settings.yaml": SETTINGS,
+    "services.yaml": SERVICES,
+    "bookmarks.yaml": BOOKMARKS,
+    "widgets.yaml": WIDGETS,
+}
+
+for filename, content in files.items():
+    path = os.path.join(CONFIG_DIR, filename)
+    with open(path, "w") as f:
+        f.write(content)
+    print(f"Written: {path}")
+
+print("All config files written.")