From 841d109689d235a11cbf4641a00216ebb332a196 Mon Sep 17 00:00:00 2001 From: Andy Dixon Date: Thu, 11 Dec 2025 22:16:54 +0000 Subject: [PATCH] Initial Commit --- .gitignore | 2 + README.md | 205 +++++++++++++++++++++++ monitor_loop.sh | 21 +++ nationalgrid_monitor.py | 350 ++++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 5 files changed, 579 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100755 monitor_loop.sh create mode 100755 nationalgrid_monitor.py create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..da702e9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +Bishops_Wood.json London.json Yorkshire.json +__pycache__ diff --git a/README.md b/README.md new file mode 100644 index 0000000..463bb99 --- /dev/null +++ b/README.md @@ -0,0 +1,205 @@ +# National Grid Real-Time Data Monitor + +A Python application that fetches real-time data from the National Grid Carbon Intensity API, including: +- Load and demand information +- Fuel mixture (generation mix) +- Carbon intensity forecasts +- Regional and national statistics + +## Features + +- Configurable Grid Supply Point (GSP) selection +- Real-time data collection from National Grid APIs +- JSON output format (perfect for Grafana integration) +- Regional generation mix data +- National statistics and carbon intensity + +## Installation + +No dependencies required beyond Python 3 standard library and `requests`: + +```bash +pip install requests +``` + +## Usage + +### Default (Bishops Wood): +```bash +python3 nationalgrid_monitor.py +``` + +### Specify a different GSP: +```bash +python3 nationalgrid_monitor.py --gsp "London" +python3 nationalgrid_monitor.py --gsp "North Scotland" +python3 nationalgrid_monitor.py --gsp "Yorkshire" +``` + +### List available GSPs: +```bash +python3 nationalgrid_monitor.py --list +``` + +## Available GSPs + +- Bishops Wood (West Midlands) +- North Scotland +- South Scotland +- North West England +- North East England +- Yorkshire +- North Wales +- South Wales +- West Midlands +- East Midlands +- East England +- South West England +- South England +- London +- South East England + +## Output + +Data is saved to `.json` in the current directory. + +Example output structure: +```json +{ + "gsp_name": "Bishops Wood", + "gsp_id": 11, + "timestamp": "2025-12-11T17:39:24.668415Z", + "collection_time": "2025-12-11T17:39:24.668436", + "regional": { + "intensity": {...}, + "generation_mix": [...] + }, + "national": { + "intensity": {...} + }, + "generation": { + "mix": [...] + } +} +``` + +## VictoriaMetrics Integration + +The tool pushes metrics to VictoriaMetrics using the Prometheus exposition format via the `/api/v1/import/prometheus` endpoint. + +### Supported Endpoints + +**Direct VictoriaMetrics** (single-node): +- Default: `http://localhost:8428/api/v1/import/prometheus` + +**vmagent** (if using vmagent as a proxy): +- Typically: `http://localhost:8429/api/v1/import/prometheus` + +**VictoriaMetrics Cluster** (vminsert): +- Format: `http://vminsert:8480/insert/0/prometheus/api/v1/import/prometheus` + +### Usage + +```bash +# Push to VictoriaMetrics single-node on localhost (default) +python3 nationalgrid_monitor.py --gsp "Bishops Wood" + +# Push to vmagent +python3 nationalgrid_monitor.py --victoriametrics http://localhost:8429 + +# Push to VictoriaMetrics cluster +python3 nationalgrid_monitor.py --victoriametrics http://vminsert:8480/insert/0/prometheus + +# Push to custom VictoriaMetrics instance +python3 nationalgrid_monitor.py --victoriametrics http://your-vm-host:8428 + +# Skip VictoriaMetrics push (JSON only) +python3 nationalgrid_monitor.py --no-push +``` + +### Metrics Format + +All metrics are exported as `national_grid{gsp="",metric=""} ` + +Available metrics: +- `demand_mw` - **Actual national demand in MW** (from Elexon BMRS API - no API key required) +- `carbon_intensity` - Actual carbon intensity (gCO2/kWh) +- `carbon_intensity_forecast` - Forecast carbon intensity (gCO2/kWh) +- `load_indicator` - Load indicator based on carbon intensity +- `wind` - Wind generation percentage +- `gas` - Gas generation percentage +- `nuclear` - Nuclear generation percentage +- `biomass` - Biomass generation percentage +- `solar` - Solar generation percentage +- `hydro` - Hydro generation percentage +- `coal` - Coal generation percentage +- `imports` - Imports percentage +- `other` - Other generation percentage + +### Continuous Monitoring + +Run the monitoring loop to continuously push data: + +```bash +# Default: Bishops Wood, 1-minute intervals +./monitor_loop.sh + +# Custom GSP and interval (seconds) +./monitor_loop.sh "London" 60 + +# Custom interval (e.g., 5 minutes) +./monitor_loop.sh "Bishops Wood" 300 + +# Custom VictoriaMetrics URL +./monitor_loop.sh "London" 60 "http://10.0.0.5:8428" +``` + +## Grafana Integration + +Once metrics are in VictoriaMetrics: + +1. Add VictoriaMetrics as a Prometheus datasource in Grafana +2. Query metrics like: `national_grid{gsp="Bishops_Wood",metric="wind"}` +3. Create dashboards with panels for: + - Generation mix (stacked graph) + - Carbon intensity (time series) + - Individual fuel sources (gauges) + - Load/demand trends + +### Example PromQL Queries + +```promql +# Wind generation for Bishops Wood +national_grid{gsp="Bishops_Wood",metric="wind"} + +# Carbon intensity across all GSPs +national_grid{metric="carbon_intensity"} + +# Gas generation comparison +national_grid{metric="gas"} + +# Total renewable (wind + solar + hydro) +national_grid{metric="wind"} + national_grid{metric="solar"} + national_grid{metric="hydro"} +``` + +## API Data Sources + +This tool uses multiple free APIs (no API key required): + +### Carbon Intensity API +- Real-time carbon intensity data +- Generation mix by fuel type (biomass, coal, gas, nuclear, wind, solar, etc.) +- Regional and national statistics +- Half-hourly forecasts (minute resolution available) +- API: https://api.carbonintensity.org.uk/ +- Resolution: minute (for most recent data) + +### Elexon BMRS API +- **Actual national demand in MW** (updated every 5 minutes) +- No API key required for demand data +- API: https://data.elexon.co.uk/bmrs/api/v1/demand/outturn/summary +- Resolution: minute (for latest data) + +## License + +MIT diff --git a/monitor_loop.sh b/monitor_loop.sh new file mode 100755 index 0000000..13f071f --- /dev/null +++ b/monitor_loop.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Continuous monitoring script for VictoriaMetrics/Grafana integration +# Fetches data every minute and pushes to VictoriaMetrics + +GSP_NAME="${1:-Bishops Wood}" +INTERVAL="${2:-60}" # 1 minute default +VM_URL="${3:-http://localhost:8428}" + +echo "Starting continuous monitoring for GSP: $GSP_NAME" +echo "Update interval: $INTERVAL seconds" +echo "VictoriaMetrics URL: $VM_URL" +echo "Press Ctrl+C to stop" +echo "" + +while true; do + echo "[$(date)] Fetching data..." + python3 nationalgrid_monitor.py --gsp "$GSP_NAME" --victoriametrics "$VM_URL" + echo "Sleeping for $INTERVAL seconds..." + echo "" + sleep "$INTERVAL" +done diff --git a/nationalgrid_monitor.py b/nationalgrid_monitor.py new file mode 100755 index 0000000..8f402c5 --- /dev/null +++ b/nationalgrid_monitor.py @@ -0,0 +1,350 @@ +#!/usr/bin/env python3 +""" +National Grid Real-Time Data Monitor +Fetches load, fuel mixture, and other real-time data for a configurable GSP +Pushes metrics to VictoriaMetrics Prometheus endpoint +""" + +import requests +import json +import sys +from datetime import datetime +from typing import Dict, Any, Optional, List + +# GSP ID mapping (based on Carbon Intensity API regional IDs) +GSP_MAPPING = { + "Bishops Wood": 11, # West Midlands + "North Scotland": 1, + "South Scotland": 2, + "North West England": 3, + "North East England": 4, + "Yorkshire": 5, + "North Wales": 6, + "South Wales": 7, + "West Midlands": 11, + "East Midlands": 12, + "East England": 13, + "South West England": 14, + "South England": 15, + "London": 16, + "South East England": 17, +} + +class NationalGridMonitor: + """Monitor for National Grid real-time data""" + + BASE_URL = "https://api.carbonintensity.org.uk" + + def __init__(self, gsp_name: str = "Bishops Wood", victoriametrics_url: str = None): + self.gsp_name = gsp_name + self.gsp_id = GSP_MAPPING.get(gsp_name) + self.victoriametrics_url = victoriametrics_url + + if self.gsp_id is None: + print(f"Warning: GSP '{gsp_name}' not found in mapping. Available GSPs:") + for name in sorted(GSP_MAPPING.keys()): + print(f" - {name}") + sys.exit(1) + + def get_regional_data(self) -> Optional[Dict[str, Any]]: + """Fetch regional data including generation mix and intensity""" + try: + url = f"{self.BASE_URL}/regional/regionid/{self.gsp_id}" + params = {"resolution": "minute"} + response = requests.get(url, params=params, timeout=10) + response.raise_for_status() + return response.json() + except requests.exceptions.RequestException as e: + print(f"Error fetching regional data: {e}") + return None + + def get_national_stats(self) -> Optional[Dict[str, Any]]: + """Fetch national statistics including demand""" + try: + url = f"{self.BASE_URL}/intensity" + params = {"resolution": "minute"} + response = requests.get(url, params=params, timeout=10) + response.raise_for_status() + return response.json() + except requests.exceptions.RequestException as e: + print(f"Error fetching national stats: {e}") + return None + + def get_generation_mix(self) -> Optional[Dict[str, Any]]: + """Fetch current generation mix""" + try: + url = f"{self.BASE_URL}/generation" + params = {"resolution": "minute"} + response = requests.get(url, params=params, timeout=10) + response.raise_for_status() + return response.json() + except requests.exceptions.RequestException as e: + print(f"Error fetching generation mix: {e}") + return None + + def get_demand_data(self) -> Optional[Dict[str, Any]]: + """Fetch actual demand data from Elexon BMRS API (no API key required)""" + try: + # Elexon BMRS API provides actual demand in MW + url = "https://data.elexon.co.uk/bmrs/api/v1/demand/outturn/summary" + params = {"resolution": "minute"} + response = requests.get(url, params=params, timeout=10) + response.raise_for_status() + return response.json() + except requests.exceptions.RequestException as e: + print(f"Error fetching demand data: {e}") + return None + + def collect_data(self) -> Dict[str, Any]: + """Collect all available data and format for output""" + + print(f"Fetching data for GSP: {self.gsp_name} (ID: {self.gsp_id})...") + + regional_data = self.get_regional_data() + national_stats = self.get_national_stats() + generation_mix = self.get_generation_mix() + demand_data = self.get_demand_data() + + # Build comprehensive data structure + output = { + "gsp_name": self.gsp_name, + "gsp_id": self.gsp_id, + "timestamp": datetime.now(datetime.UTC if hasattr(datetime, 'UTC') else None).replace(tzinfo=None).isoformat() + "Z", + "collection_time": datetime.now().isoformat(), + } + + # Regional data + if regional_data and "data" in regional_data: + data_entry = regional_data["data"][0] if regional_data["data"] else {} + output["regional"] = { + "from": data_entry.get("from"), + "to": data_entry.get("to"), + "intensity": data_entry.get("intensity", {}), + "generation_mix": data_entry.get("generationmix", []) + } + + # National statistics + if national_stats and "data" in national_stats: + data_entry = national_stats["data"][0] if national_stats["data"] else {} + output["national"] = { + "from": data_entry.get("from"), + "to": data_entry.get("to"), + "intensity": data_entry.get("intensity", {}) + } + + # National generation mix + if generation_mix and "data" in generation_mix: + data_entry = generation_mix["data"] + output["generation"] = { + "from": data_entry.get("from"), + "to": data_entry.get("to"), + "mix": data_entry.get("generationmix", []) + } + + # Actual demand from Elexon BMRS API (in MW) + if demand_data and isinstance(demand_data, list) and len(demand_data) > 0: + # Get the most recent demand reading + latest_demand = demand_data[-1] + output["demand"] = { + "timestamp": latest_demand.get("startTime"), + "demand_mw": latest_demand.get("demand"), + "record_type": latest_demand.get("recordType"), + "source": "Elexon BMRS API" + } + else: + output["demand"] = { + "demand_mw": None, + "note": "Demand data unavailable" + } + + return output + + def save_to_file(self, data: Dict[str, Any]) -> None: + """Save data to JSON file""" + filename = f"{self.gsp_name.replace(' ', '_')}.json" + + with open(filename, 'w') as f: + json.dump(data, f, indent=2) + + print(f"Data saved to: {filename}") + + def format_prometheus_metrics(self, data: Dict[str, Any]) -> str: + """Format data as Prometheus exposition format for VictoriaMetrics""" + metrics = [] + gsp_label = self.gsp_name.replace(' ', '_').replace('-', '_') + timestamp_ms = int(datetime.now().timestamp() * 1000) + + # Carbon intensity + if "national" in data and "intensity" in data["national"]: + intensity = data["national"]["intensity"] + if "actual" in intensity and intensity["actual"] is not None: + metrics.append( + f'national_grid{{gsp="{gsp_label}",metric="carbon_intensity"}} {intensity["actual"]} {timestamp_ms}' + ) + if "forecast" in intensity and intensity["forecast"] is not None: + metrics.append( + f'national_grid{{gsp="{gsp_label}",metric="carbon_intensity_forecast"}} {intensity["forecast"]} {timestamp_ms}' + ) + + # Generation mix percentages + if "generation" in data and "mix" in data["generation"]: + for fuel_data in data["generation"]["mix"]: + fuel = fuel_data["fuel"] + perc = fuel_data["perc"] + metrics.append( + f'national_grid{{gsp="{gsp_label}",metric="{fuel}"}} {perc} {timestamp_ms}' + ) + + # Regional generation mix (if available) + if "regional" in data and "generation_mix" in data["regional"]: + for fuel_data in data["regional"]["generation_mix"]: + fuel = fuel_data["fuel"] + perc = fuel_data["perc"] + metrics.append( + f'national_grid{{gsp="{gsp_label}",metric="regional_{fuel}"}} {perc} {timestamp_ms}' + ) + + # Regional intensity (if available) + if "regional" in data and "intensity" in data["regional"]: + intensity = data["regional"]["intensity"] + if "forecast" in intensity and intensity["forecast"] is not None: + metrics.append( + f'national_grid{{gsp="{gsp_label}",metric="regional_carbon_intensity"}} {intensity["forecast"]} {timestamp_ms}' + ) + + # Actual demand in MW from Elexon + if "demand" in data and "demand_mw" in data["demand"]: + demand_mw = data["demand"]["demand_mw"] + if demand_mw is not None: + metrics.append( + f'national_grid{{gsp="{gsp_label}",metric="demand_mw"}} {demand_mw} {timestamp_ms}' + ) + + # Carbon intensity as a load indicator + if "national" in data and "intensity" in data["national"]: + actual_intensity = data["national"]["intensity"].get("actual") + if actual_intensity is not None: + metrics.append( + f'national_grid{{gsp="{gsp_label}",metric="load_indicator"}} {actual_intensity} {timestamp_ms}' + ) + + return "\n".join(metrics) + "\n" + + def push_to_victoriametrics(self, data: Dict[str, Any]) -> bool: + """Push metrics to VictoriaMetrics using Prometheus exposition format""" + if not self.victoriametrics_url: + return False + + try: + metrics_text = self.format_prometheus_metrics(data) + + # VictoriaMetrics supports multiple ingestion endpoints: + # 1. /api/v1/import/prometheus - VictoriaMetrics native (Prometheus text format) + # 2. /api/v1/write - Prometheus remote_write protocol (requires protobuf) + # Using native import endpoint for simplicity + url = f"{self.victoriametrics_url}/api/v1/import/prometheus" + + headers = { + "Content-Type": "text/plain" + } + + response = requests.post(url, data=metrics_text, headers=headers, timeout=10) + response.raise_for_status() + + print(f"✓ Pushed {len(metrics_text.strip().split(chr(10)))} metrics to VictoriaMetrics: {url}") + return True + + except requests.exceptions.RequestException as e: + print(f"✗ Error pushing to VictoriaMetrics: {e}") + return False + + def run(self) -> None: + """Main execution method""" + data = self.collect_data() + self.save_to_file(data) + + # Push to VictoriaMetrics if configured + if self.victoriametrics_url: + self.push_to_victoriametrics(data) + + # Print summary + print("\n=== Data Summary ===") + + # National intensity + if "national" in data and "intensity" in data["national"]: + intensity = data["national"]["intensity"] + print(f"National Carbon Intensity: {intensity.get('actual', 'N/A')} gCO2/kWh (forecast: {intensity.get('forecast', 'N/A')})") + print(f"Index: {intensity.get('index', 'N/A')}") + + # Regional intensity + if "regional" in data and "intensity" in data["regional"] and data["regional"]["intensity"]: + print(f"Regional Intensity: {data['regional']['intensity'].get('forecast')} gCO2/kWh") + print(f"Regional Index: {data['regional']['intensity'].get('index')}") + + # National generation mix + if "generation" in data: + print("\nNational Generation Mix:") + for gen in data["generation"]["mix"]: + print(f" {gen['fuel']:12s}: {gen['perc']:5.1f}%") + + # Regional generation mix (if different/available) + if "regional" in data and data["regional"].get("generation_mix"): + print("\nRegional Generation Mix:") + for gen in data["regional"]["generation_mix"]: + print(f" {gen['fuel']:12s}: {gen['perc']:5.1f}%") + + # Demand info + if "demand" in data: + demand_mw = data['demand'].get('demand_mw', 'N/A') + if demand_mw != 'N/A' and demand_mw is not None: + print(f"\nNational Demand: {demand_mw:,.0f} MW") + print(f"Source: {data['demand'].get('source', 'Unknown')}") + if "timestamp" in data['demand']: + print(f"Timestamp: {data['demand']['timestamp']}") + else: + print(f"\nDemand: {data['demand'].get('note', 'N/A')}") + + +def main(): + """Main entry point""" + import argparse + + parser = argparse.ArgumentParser( + description="Fetch National Grid real-time data for a GSP and push to VictoriaMetrics" + ) + parser.add_argument( + "--gsp", + default="Bishops Wood", + help="Grid Supply Point name (default: Bishops Wood)" + ) + parser.add_argument( + "--victoriametrics", + default="http://localhost:8428", + help="VictoriaMetrics URL (default: http://localhost:8428)" + ) + parser.add_argument( + "--no-push", + action="store_true", + help="Don't push to VictoriaMetrics, only save to JSON" + ) + parser.add_argument( + "--list", + action="store_true", + help="List available GSP names" + ) + + args = parser.parse_args() + + if args.list: + print("Available GSP names:") + for name in sorted(GSP_MAPPING.keys()): + print(f" - {name}") + sys.exit(0) + + vm_url = None if args.no_push else args.victoriametrics + monitor = NationalGridMonitor(gsp_name=args.gsp, victoriametrics_url=vm_url) + monitor.run() + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0eb8cae --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +requests>=2.31.0