Benchmark Sourcecode
Contents
Benchmark Sourcecode#
Comparing the results of NEMED to AEMO CDEII Reporting for Regional Sent-Out Generation, Total Emissions and Average Emissions Intensity
Import Packages#
Show code cell content
import nemed
from nemed.downloader import download_aemo_cdeii_summary, download_unit_dispatch
from nemed.process import get_total_emissions_by_DI_DUID, aggregate_data_by
# To generate plots shown
import pandas as pd
import numpy as np
import os
import plotly.graph_objects as go
import plotly.express as px
from plotly.subplots import make_subplots
# Open plot in browser (optional)
import plotly.io as pio
pio.renderers.default = "browser"
# Hide NEMOSIS logging
import logging
logging.getLogger("nemosis").setLevel(logging.WARNING)
Extracting & Processing Emissions Data#
Refer to explained details in the Total Emissions Example
Warning
Data size and computation time is extensive for a 7 year period.
Computation time for get_total_emissions was approx 1 hr.
Cached directory consumes approx 18GB.
start = "2015/01/01 00:00"
end = "2022/01/01 00:00"
cache = "E:/TEMPCACHE_nemed_demo/"
result = nemed.get_total_emissions(start_time=start,
end_time=end,
cache=cache,
filter_regions=None,
by=None,
generation_sent_out=True,
assume_energy_ramp=True,
return_pivot=False)
result.to_csv(r'E:\TEMPCACHE_nemed_demo\EXAMPLE_BENCHMARK_DATA_ASOF_09012023.CSV')
Show code cell output
INFO: Processing total emissions from 2014-12-31 to 2015-01-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2015-01-01 to 2015-02-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2015-02-01 to 2015-03-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2015-03-01 to 2015-04-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2015-04-01 to 2015-05-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2015-05-01 to 2015-06-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2015-06-01 to 2015-07-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2015-07-01 to 2015-08-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2015-08-01 to 2015-09-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2015-09-01 to 2015-10-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2015-10-01 to 2015-11-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2015-11-01 to 2015-12-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2015-12-01 to 2016-01-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2016-01-01 to 2016-02-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2016-02-01 to 2016-03-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2016-03-01 to 2016-04-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2016-04-01 to 2016-05-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2016-05-01 to 2016-06-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2016-06-01 to 2016-07-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2016-07-01 to 2016-08-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2016-08-01 to 2016-09-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2016-09-01 to 2016-10-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2016-10-01 to 2016-11-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2016-11-01 to 2016-12-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2016-12-01 to 2017-01-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2017-01-01 to 2017-02-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2017-02-01 to 2017-03-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2017-03-01 to 2017-04-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2017-04-01 to 2017-05-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2017-05-01 to 2017-06-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2017-06-01 to 2017-07-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2017-07-01 to 2017-08-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2017-08-01 to 2017-09-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2017-09-01 to 2017-10-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2017-10-01 to 2017-11-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2017-11-01 to 2017-12-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2017-12-01 to 2018-01-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2018-01-01 to 2018-02-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2018-02-01 to 2018-03-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2018-03-01 to 2018-04-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2018-04-01 to 2018-05-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2018-05-01 to 2018-06-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2018-06-01 to 2018-07-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2018-07-01 to 2018-08-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2018-08-01 to 2018-09-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2018-09-01 to 2018-10-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2018-10-01 to 2018-11-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2018-11-01 to 2018-12-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2018-12-01 to 2019-01-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2019-01-01 to 2019-02-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2019-02-01 to 2019-03-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2019-03-01 to 2019-04-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2019-04-01 to 2019-05-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2019-05-01 to 2019-06-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2019-06-01 to 2019-07-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2019-07-01 to 2019-08-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2019-08-01 to 2019-09-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2019-09-01 to 2019-10-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2019-10-01 to 2019-11-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2019-11-01 to 2019-12-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2019-12-01 to 2020-01-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2020-01-01 to 2020-02-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2020-02-01 to 2020-03-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2020-03-01 to 2020-04-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2020-04-01 to 2020-05-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2020-05-01 to 2020-06-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2020-06-01 to 2020-07-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2020-07-01 to 2020-08-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2020-08-01 to 2020-09-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2020-09-01 to 2020-10-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2020-10-01 to 2020-11-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2020-11-01 to 2020-12-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2020-12-01 to 2021-01-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2021-01-01 to 2021-02-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2021-02-01 to 2021-03-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2021-03-01 to 2021-04-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2021-04-01 to 2021-05-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2021-05-01 to 2021-06-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2021-06-01 to 2021-07-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2021-07-01 to 2021-08-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2021-08-01 to 2021-09-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2021-09-01 to 2021-10-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2021-10-01 to 2021-11-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2021-11-01 to 2021-12-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Processing total emissions from 2021-12-01 to 2022-01-01
INFO: Compiling Energy from Dispatch
INFO: Compiling Sent Out Generation
INFO: Loading results file processed_co2_total_2014-12-31_2015-01-01.parquet
INFO: Loading results file processed_co2_total_2015-01-01_2015-02-01.parquet
INFO: Loading results file processed_co2_total_2015-02-01_2015-03-01.parquet
INFO: Loading results file processed_co2_total_2015-03-01_2015-04-01.parquet
INFO: Loading results file processed_co2_total_2015-04-01_2015-05-01.parquet
INFO: Loading results file processed_co2_total_2015-05-01_2015-06-01.parquet
INFO: Loading results file processed_co2_total_2015-06-01_2015-07-01.parquet
INFO: Loading results file processed_co2_total_2015-07-01_2015-08-01.parquet
INFO: Loading results file processed_co2_total_2015-08-01_2015-09-01.parquet
INFO: Loading results file processed_co2_total_2015-09-01_2015-10-01.parquet
INFO: Loading results file processed_co2_total_2015-10-01_2015-11-01.parquet
INFO: Loading results file processed_co2_total_2015-11-01_2015-12-01.parquet
INFO: Loading results file processed_co2_total_2015-12-01_2016-01-01.parquet
INFO: Loading results file processed_co2_total_2016-01-01_2016-02-01.parquet
INFO: Loading results file processed_co2_total_2016-02-01_2016-03-01.parquet
INFO: Loading results file processed_co2_total_2016-03-01_2016-04-01.parquet
INFO: Loading results file processed_co2_total_2016-04-01_2016-05-01.parquet
INFO: Loading results file processed_co2_total_2016-05-01_2016-06-01.parquet
INFO: Loading results file processed_co2_total_2016-06-01_2016-07-01.parquet
INFO: Loading results file processed_co2_total_2016-07-01_2016-08-01.parquet
INFO: Loading results file processed_co2_total_2016-08-01_2016-09-01.parquet
INFO: Loading results file processed_co2_total_2016-09-01_2016-10-01.parquet
INFO: Loading results file processed_co2_total_2016-10-01_2016-11-01.parquet
INFO: Loading results file processed_co2_total_2016-11-01_2016-12-01.parquet
INFO: Loading results file processed_co2_total_2016-12-01_2017-01-01.parquet
INFO: Loading results file processed_co2_total_2017-01-01_2017-02-01.parquet
INFO: Loading results file processed_co2_total_2017-02-01_2017-03-01.parquet
INFO: Loading results file processed_co2_total_2017-03-01_2017-04-01.parquet
INFO: Loading results file processed_co2_total_2017-04-01_2017-05-01.parquet
INFO: Loading results file processed_co2_total_2017-05-01_2017-06-01.parquet
INFO: Loading results file processed_co2_total_2017-06-01_2017-07-01.parquet
INFO: Loading results file processed_co2_total_2017-07-01_2017-08-01.parquet
INFO: Loading results file processed_co2_total_2017-08-01_2017-09-01.parquet
INFO: Loading results file processed_co2_total_2017-09-01_2017-10-01.parquet
INFO: Loading results file processed_co2_total_2017-10-01_2017-11-01.parquet
INFO: Loading results file processed_co2_total_2017-11-01_2017-12-01.parquet
INFO: Loading results file processed_co2_total_2017-12-01_2018-01-01.parquet
INFO: Loading results file processed_co2_total_2018-01-01_2018-02-01.parquet
INFO: Loading results file processed_co2_total_2018-02-01_2018-03-01.parquet
INFO: Loading results file processed_co2_total_2018-03-01_2018-04-01.parquet
INFO: Loading results file processed_co2_total_2018-04-01_2018-05-01.parquet
INFO: Loading results file processed_co2_total_2018-05-01_2018-06-01.parquet
INFO: Loading results file processed_co2_total_2018-06-01_2018-07-01.parquet
INFO: Loading results file processed_co2_total_2018-07-01_2018-08-01.parquet
INFO: Loading results file processed_co2_total_2018-08-01_2018-09-01.parquet
INFO: Loading results file processed_co2_total_2018-09-01_2018-10-01.parquet
INFO: Loading results file processed_co2_total_2018-10-01_2018-11-01.parquet
INFO: Loading results file processed_co2_total_2018-11-01_2018-12-01.parquet
INFO: Loading results file processed_co2_total_2018-12-01_2019-01-01.parquet
INFO: Loading results file processed_co2_total_2019-01-01_2019-02-01.parquet
INFO: Loading results file processed_co2_total_2019-02-01_2019-03-01.parquet
INFO: Loading results file processed_co2_total_2019-03-01_2019-04-01.parquet
INFO: Loading results file processed_co2_total_2019-04-01_2019-05-01.parquet
INFO: Loading results file processed_co2_total_2019-05-01_2019-06-01.parquet
INFO: Loading results file processed_co2_total_2019-06-01_2019-07-01.parquet
INFO: Loading results file processed_co2_total_2019-07-01_2019-08-01.parquet
INFO: Loading results file processed_co2_total_2019-08-01_2019-09-01.parquet
INFO: Loading results file processed_co2_total_2019-09-01_2019-10-01.parquet
INFO: Loading results file processed_co2_total_2019-10-01_2019-11-01.parquet
INFO: Loading results file processed_co2_total_2019-11-01_2019-12-01.parquet
INFO: Loading results file processed_co2_total_2019-12-01_2020-01-01.parquet
INFO: Loading results file processed_co2_total_2020-01-01_2020-02-01.parquet
INFO: Loading results file processed_co2_total_2020-02-01_2020-03-01.parquet
INFO: Loading results file processed_co2_total_2020-03-01_2020-04-01.parquet
INFO: Loading results file processed_co2_total_2020-04-01_2020-05-01.parquet
INFO: Loading results file processed_co2_total_2020-05-01_2020-06-01.parquet
INFO: Loading results file processed_co2_total_2020-06-01_2020-07-01.parquet
INFO: Loading results file processed_co2_total_2020-07-01_2020-08-01.parquet
INFO: Loading results file processed_co2_total_2020-08-01_2020-09-01.parquet
INFO: Loading results file processed_co2_total_2020-09-01_2020-10-01.parquet
INFO: Loading results file processed_co2_total_2020-10-01_2020-11-01.parquet
INFO: Loading results file processed_co2_total_2020-11-01_2020-12-01.parquet
INFO: Loading results file processed_co2_total_2020-12-01_2021-01-01.parquet
INFO: Loading results file processed_co2_total_2021-01-01_2021-02-01.parquet
INFO: Loading results file processed_co2_total_2021-02-01_2021-03-01.parquet
INFO: Loading results file processed_co2_total_2021-03-01_2021-04-01.parquet
INFO: Loading results file processed_co2_total_2021-04-01_2021-05-01.parquet
INFO: Loading results file processed_co2_total_2021-05-01_2021-06-01.parquet
INFO: Loading results file processed_co2_total_2021-06-01_2021-07-01.parquet
INFO: Loading results file processed_co2_total_2021-07-01_2021-08-01.parquet
INFO: Loading results file processed_co2_total_2021-08-01_2021-09-01.parquet
INFO: Loading results file processed_co2_total_2021-09-01_2021-10-01.parquet
INFO: Loading results file processed_co2_total_2021-10-01_2021-11-01.parquet
INFO: Loading results file processed_co2_total_2021-11-01_2021-12-01.parquet
INFO: Loading results file processed_co2_total_2021-12-01_2022-01-01.parquet
INFO: Completed get_total_emissions_by_DI_DUID
Tip
A .csv file containing the above processed data can be saved for convenience and reloaded as below.
result = pd.read_csv(r'C:\Users\derlu\Downloads'+\
'\EXAMPLE_BENCHMARK_DATA_ASOF_09012023.CSV',index_col=[0])
result['TimeEnding'] = pd.to_datetime(result['TimeEnding'], format="%Y-%m-%d %H:%M:%S")
result
| TimeEnding | Region | Energy | Total_Emissions | Intensity_Index | |
|---|---|---|---|---|---|
| 0 | 2015-01-01 00:05:00 | NEM | 0.000000 | 0.000000 | 0.000000 |
| 1 | 2015-01-01 00:05:00 | NSW1 | 0.000000 | 0.000000 | 0.000000 |
| 2 | 2015-01-01 00:05:00 | QLD1 | 0.000000 | 0.000000 | 0.000000 |
| 3 | 2015-01-01 00:05:00 | SA1 | 0.000000 | 0.000000 | 0.000000 |
| 4 | 2015-01-01 00:05:00 | TAS1 | 0.000000 | 0.000000 | 0.000000 |
| ... | ... | ... | ... | ... | ... |
| 4418491 | 2022-01-01 00:00:00 | NSW1 | 526.126887 | 381.533186 | 0.725173 |
| 4418492 | 2022-01-01 00:00:00 | QLD1 | 538.572738 | 398.250305 | 0.739455 |
| 4418493 | 2022-01-01 00:00:00 | SA1 | 103.334027 | 35.780752 | 0.346263 |
| 4418494 | 2022-01-01 00:00:00 | TAS1 | 138.785120 | 0.000000 | 0.000000 |
| 4418495 | 2022-01-01 00:00:00 | VIC1 | 483.276172 | 389.073125 | 0.805074 |
4418496 rows × 5 columns
Data Aggregation#
nemed_df = aggregate_data_by(result, "day")
nemed_df
| TimeBeginning | TimeEnding | Region | Energy | Total_Emissions | Intensity_Index | |
|---|---|---|---|---|---|---|
| 0 | 2015-01-01 | 2015-01-02 | NEM | 499240.152 | 446855.903 | 0.895 |
| 1 | 2015-01-02 | 2015-01-03 | NEM | 554330.260 | 474310.122 | 0.856 |
| 2 | 2015-01-03 | 2015-01-04 | NEM | 553499.542 | 482764.901 | 0.872 |
| 3 | 2015-01-04 | 2015-01-05 | NEM | 499399.915 | 445382.707 | 0.892 |
| 4 | 2015-01-05 | 2015-01-06 | NEM | 533848.633 | 481068.411 | 0.901 |
| ... | ... | ... | ... | ... | ... | ... |
| 15337 | 2021-12-27 | 2021-12-28 | VIC1 | 105752.913 | 70330.291 | 0.665 |
| 15338 | 2021-12-28 | 2021-12-29 | VIC1 | 110167.130 | 82720.976 | 0.751 |
| 15339 | 2021-12-29 | 2021-12-30 | VIC1 | 114128.214 | 99348.784 | 0.871 |
| 15340 | 2021-12-30 | 2021-12-31 | VIC1 | 125026.680 | 104753.933 | 0.838 |
| 15341 | 2021-12-31 | 2022-01-01 | VIC1 | 145208.124 | 105244.182 | 0.725 |
15342 rows × 6 columns
Retrieve Published AEMO CDEII data for comparison#
This function extracts the publicly available ‘Summary Results’ files from the AEMO CDEII website.
aemo_df = download_aemo_cdeii_summary(filter_start=start,
filter_end=end,
cache=cache)
Show code cell output
WARNING: 'filter_end' date exceeds available data from CURRENT CDEII. Latest available data is 2021-12-31 00:00:00
aemo_df.head()
| SETTLEMENTDATE | REGIONID | TOTAL_SENT_OUT_ENERGY | TOTAL_EMISSIONS | CO2E_INTENSITY_INDEX | |
|---|---|---|---|---|---|
| 0 | 2015-01-01 | NEM | 477475.87550 | 444620.356900 | 0.9312 |
| 1 | 2015-01-01 | NSW1 | 134533.32030 | 113619.721600 | 0.8445 |
| 2 | 2015-01-01 | QLD1 | 162723.14120 | 140410.466200 | 0.8629 |
| 3 | 2015-01-01 | SA1 | 28239.80165 | 17266.490910 | 0.6114 |
| 4 | 2015-01-01 | TAS1 | 18188.65593 | 2.103155 | 0.0001 |
Compute Error between Datasets#
The following function calculate the error for each dataset (energy, total emissions, emissions intensity).
def calculate_dataset_errors(nemed_df, aemo_df):
all_df = pd.merge(left=nemed_df, left_on=["TimeBeginning", "Region"],
right=aemo_df, right_on=["SETTLEMENTDATE", "REGIONID"], how="inner")
all_df['ERR_Energy'] = 100 * (all_df['Energy'] - all_df['TOTAL_SENT_OUT_ENERGY'])\
/ all_df['TOTAL_SENT_OUT_ENERGY']
all_df['ERR_Total_Emissions'] = 100 * (all_df['Total_Emissions'] \
- all_df['TOTAL_EMISSIONS']) / all_df['TOTAL_EMISSIONS']
all_df['ERR_Intensity_Index'] = 100 * (all_df['Intensity_Index'] \
- all_df['CO2E_INTENSITY_INDEX']) / all_df['CO2E_INTENSITY_INDEX']
return all_df[['TimeBeginning','Region','Energy','TOTAL_SENT_OUT_ENERGY',\
'ERR_Energy','Total_Emissions','TOTAL_EMISSIONS','ERR_Total_Emissions',\
'Intensity_Index','CO2E_INTENSITY_INDEX','ERR_Intensity_Index']]
plt_df = calculate_dataset_errors(nemed_df, aemo_df)
The energy metrics of the dataframe are produced:
plt_df.iloc[0:5,0:5]
| TimeBeginning | Region | Energy | TOTAL_SENT_OUT_ENERGY | ERR_Energy | |
|---|---|---|---|---|---|
| 0 | 2015-01-01 | NEM | 499240.152 | 477475.8755 | 4.558194 |
| 1 | 2015-01-02 | NEM | 554330.260 | 533685.2756 | 3.868382 |
| 2 | 2015-01-03 | NEM | 553499.542 | 525521.7957 | 5.323803 |
| 3 | 2015-01-04 | NEM | 499399.915 | 473576.6095 | 5.452825 |
| 4 | 2015-01-05 | NEM | 533848.633 | 506759.7498 | 5.345508 |
The next columns contain the emissions metrics:
plt_df.iloc[0:5,5:8]
| Total_Emissions | TOTAL_EMISSIONS | ERR_Total_Emissions | |
|---|---|---|---|
| 0 | 446855.903 | 444620.3569 | 0.502799 |
| 1 | 474310.122 | 472317.5742 | 0.421866 |
| 2 | 482764.901 | 476553.9580 | 1.303303 |
| 3 | 445382.707 | 439177.2453 | 1.412974 |
| 4 | 481068.411 | 475037.6377 | 1.269536 |
The final columns with the intensity metrics:
plt_df.iloc[0:5,5:8]
| Total_Emissions | TOTAL_EMISSIONS | ERR_Total_Emissions | |
|---|---|---|---|
| 0 | 446855.903 | 444620.3569 | 0.502799 |
| 1 | 474310.122 | 472317.5742 | 0.421866 |
| 2 | 482764.901 | 476553.9580 | 1.303303 |
| 3 | 445382.707 | 439177.2453 | 1.412974 |
| 4 | 481068.411 | 475037.6377 | 1.269536 |
Plot Formatting#
Toggle the below code block to see the plot code
Show code cell content
colors = ['#676765','#3b4252','#af7635','#8F6593','#4F759B', '#638b66','#b65551']
def NORD_theme():
plotly_NORD_theme = pio.templates["simple_white"]
plotly_NORD_theme.layout.plot_bgcolor = "#f4f4f5"
plotly_NORD_theme.layout.paper_bgcolor = "#FFFFFF"
plotly_NORD_theme.layout.xaxis.gridcolor = '#d8dee9'
plotly_NORD_theme.layout.yaxis.gridcolor = '#d8dee9'
return plotly_NORD_theme
def set_font_size(layout, font_size=16):
layout['titlefont']['size'] = font_size + 4
layout.legend['font']['size'] = font_size
for ax in [item for item in layout if item.__contains__('xaxis')]:
layout[ax].titlefont.size = font_size
layout[ax].tickfont.size = font_size
for ax in [item for item in layout if item.__contains__('yaxis')]:
layout[ax].titlefont.size = font_size
layout[ax].tickfont.size = font_size
def plot_energy(df, region, color_idx, yrange, ytickvals, yrange_err, ytickvals_err, save=None):
sel_df = df[df['Region']==region].sort_values('TimeBeginning').reset_index(drop=True)
fig = make_subplots(rows=4, cols=1, shared_xaxes=True, vertical_spacing=0.05,
specs=[[{"rowspan": 3}],[{}],[{}],[{"rowspan":1}]])
# Traces
fig.add_trace(go.Scatter(x=sel_df['TimeBeginning'],
y=sel_df['TOTAL_SENT_OUT_ENERGY'],
name="CDEII Reported",
mode="lines+markers",
line_color=colors[0]), row=1, col=1)
fig.add_trace(go.Scatter(x=sel_df['TimeBeginning'],
y=sel_df['Energy'],
name="NEMED Calculated",
mode="lines+markers",
line_color=colors[color_idx]), row=1, col=1)
fig.add_trace(go.Scatter(x=sel_df['TimeBeginning'],
y=sel_df['ERR_Energy'],
name="Error",
mode="lines",
line_color=colors[color_idx],
line_dash='dot'), row=4, col=1)
# Layout
fig.update_layout(title=f"{region} Historical Daily Sent Out Generation<br>"+\
"<sub>NEMED | Average Emissions Methodology | CDEII Benchmark Comparison</sub>",
template=NORD_theme(),
legend={'title':'Dataseries', 'orientation':'h', 'xanchor': 'center', 'x': 0.5, 'y':-0.1},
hovermode="x")
fig.update_yaxes(title_text="Total Energy<br>(MWh)", mirror=True, showgrid=True,
autorange=False, range=yrange, tickvals=[i*10**3 for i in ytickvals], row=1, col=1)
fig.update_yaxes(title_text="Error wrt.<br>CDEII (%)", mirror=True, showgrid=True,
autorange=False, range=yrange_err, tickvals=[i for i in ytickvals_err], row=4, col=1)
fig.update_xaxes(title_text="Date (Day)", mirror=True, row=4, col=1)
fig.update_xaxes(mirror=True, row=1, col=1)
FONT_SIZE = 16
FONT_STYLE = "Raleway"
fonts = dict(tickfont=dict(size=FONT_SIZE, family=FONT_STYLE),
titlefont=dict(size=FONT_SIZE, family=FONT_STYLE))
fig.update_layout(xaxis=fonts, yaxis=fonts, yaxis4=fonts, xaxis4=fonts,
legend=dict(font=dict(size=FONT_SIZE-2, family=FONT_STYLE)),
title_font_family=FONT_STYLE,
title_font_size=22)
fig.update_annotations(font=dict(size=FONT_SIZE, family=FONT_STYLE))
if save != None:
fig.write_html(os.path.join(save, f"energy_{region}.html"))
else:
fig.show()
return
def plot_emissions(df, region, color_idx, yrange, ytickvals, yrange_err, ytickvals_err, save=None, err_off=False):
sel_df = df[df['Region']==region].sort_values('TimeBeginning').reset_index(drop=True)
fig = make_subplots(rows=4, cols=1, shared_xaxes=True, vertical_spacing=0.1,
specs=[[{"rowspan": 3}],[{}],[{}],[{"rowspan":1}]],)
# Traces
fig.add_trace(go.Scatter(x=sel_df['TimeBeginning'],
y=sel_df['TOTAL_EMISSIONS'],
name="CDEII Reported",
mode="lines+markers",
line_color=colors[0]), row=1, col=1)
fig.add_trace(go.Scatter(x=sel_df['TimeBeginning'],
y=sel_df['Total_Emissions'],
name="NEMED Calculated",
mode="lines+markers",
line_color=colors[color_idx]), row=1, col=1)
if not err_off:
fig.add_trace(go.Scatter(x=sel_df['TimeBeginning'],
y=sel_df['ERR_Total_Emissions'],
name="Error",
mode="lines",
line_color=colors[color_idx],
line_dash='dot'), row=4, col=1)
else:
fig.add_annotation(xref="x domain", yref="y domain", x=0.5, y=-0.3, \
text="<i>Error chart omitted where zero values occur.<i>", showarrow=False)
# Layout
fig.update_layout(title=f"{region} Historical Daily Emissions<br>"+\
"<sub>NEMED | Average Emissions Methodology | CDEII Benchmark Comparison</sub>",
template=NORD_theme(),
legend={'title':'Dataseries', 'orientation':'h', 'xanchor': 'center', 'x': 0.5, 'y':-0.1},
hovermode="x")
fig.update_yaxes(title_text="Total Emissions<br>(tCO2-e)", mirror=True, showgrid=True,
autorange=False, range=yrange, tickvals=[i*10**3 for i in ytickvals], row=1, col=1)
fig.update_yaxes(title_text="Error wrt.<br>CDEII (%)", mirror=True, showgrid=True,
autorange=False, range=yrange_err, tickvals=[i for i in ytickvals_err], row=4, col=1)
fig.update_xaxes(title_text="Date (Day)", mirror=True, row=4, col=1)
fig.update_xaxes(mirror=True, row=1, col=1)
FONT_SIZE = 16
FONT_STYLE = "Raleway"
fonts = dict(tickfont=dict(size=FONT_SIZE, family=FONT_STYLE),
titlefont=dict(size=FONT_SIZE, family=FONT_STYLE))
fig.update_layout(xaxis=fonts, yaxis=fonts, yaxis4=fonts, xaxis4=fonts,
legend=dict(font=dict(size=FONT_SIZE-2, family=FONT_STYLE)),
title_font_family=FONT_STYLE,
title_font_size=22)
fig.update_annotations(font=dict(size=FONT_SIZE, family=FONT_STYLE))
if save != None:
fig.write_html(os.path.join(save, f"emissions_{region}.html"))
else:
fig.show()
return
def plot_intensity(df, region, color_idx, yrange, ytickvals, yrange_err, ytickvals_err, save=None, err_off=False):
sel_df = df[df['Region']==region].sort_values('TimeBeginning').reset_index(drop=True)
fig = make_subplots(rows=4, cols=1, shared_xaxes=True, vertical_spacing=0.1,
specs=[[{"rowspan": 3}],[{}],[{}],[{"rowspan":1}]],)
# Traces
fig.add_trace(go.Scatter(x=sel_df['TimeBeginning'],
y=sel_df['CO2E_INTENSITY_INDEX'],
name="CDEII Reported",
mode="lines+markers",
line_color=colors[0]), row=1, col=1)
fig.add_trace(go.Scatter(x=sel_df['TimeBeginning'],
y=sel_df['Intensity_Index'],
name="NEMED Calculated",
mode="lines+markers",
line_color=colors[color_idx]), row=1, col=1)
if not err_off:
fig.add_trace(go.Scatter(x=sel_df['TimeBeginning'],
y=sel_df['ERR_Intensity_Index'],
name="Error",
mode="lines",
line_color=colors[color_idx],
line_dash='dot'), row=4, col=1)
else:
fig.add_annotation(xref="x domain", yref="y domain", x=0.5, y=-0.3, \
text="<i>Error chart omitted where zero values occur.<i>", showarrow=False)
# Layout
fig.update_layout(title=f"{region} Historical Daily Emissions Intensity<br>"+\
"<sub>NEMED | Average Emissions Methodology | CDEII Benchmark Comparison</sub>",
template=NORD_theme(),
legend={'title':'Dataseries', 'orientation':'h', 'xanchor': 'center', 'x': 0.5, 'y':-0.1},
hovermode="x")
fig.update_yaxes(title_text="Emissions Intensity<br>(tCO2-e/MWh)", mirror=True, showgrid=True,
autorange=False, range=yrange, tickvals=[i*10**-2 for i in ytickvals], row=1, col=1)
fig.update_yaxes(title_text="Error wrt.<br>CDEII (%)", mirror=True, showgrid=True,
autorange=False, range=yrange_err, tickvals=[i for i in ytickvals_err], row=4, col=1)
fig.update_xaxes(title_text="Date (Day)", mirror=True, row=4, col=1)
fig.update_xaxes(mirror=True, row=1, col=1)
FONT_SIZE = 16
FONT_STYLE = "Raleway"
fonts = dict(tickfont=dict(size=FONT_SIZE, family=FONT_STYLE),
titlefont=dict(size=FONT_SIZE, family=FONT_STYLE))
fig.update_layout(xaxis=fonts, yaxis=fonts, yaxis4=fonts, xaxis4=fonts,
legend=dict(font=dict(size=FONT_SIZE-2, family=FONT_STYLE)),
title_font_family=FONT_STYLE,
title_font_size=22)
fig.update_annotations(font=dict(size=FONT_SIZE, family=FONT_STYLE))
if save != None:
fig.write_html(os.path.join(save, f"intensity_{region}.html"))
else:
fig.show()
return
Execute Charts#
save_path =r'E:\PROJECTS\nemed-fork\NEMED\docs\source\examples\charts_benchmark'
Sent Out Energy#
plot_energy(plt_df,
region='NEM',
color_idx=1,
yrange=[380000,720000],
ytickvals=range(400,701,50),
yrange_err=[-2,12],
ytickvals_err=range(0,11,5),
save=save_path)
plot_energy(plt_df,
region='NSW1',
color_idx=2,
yrange=[90000,260000],
ytickvals=range(100,251,25),
yrange_err=[-2,12],
ytickvals_err=range(0,11,5),
save=save_path)
plot_energy(plt_df,
region='QLD1',
color_idx=3,
yrange=[110000,210000],
ytickvals=range(120,201,20),
yrange_err=[-10,10],
ytickvals_err=range(-5,6,5),
save=save_path)
plot_energy(plt_df,
region='SA1',
color_idx=4,
yrange=[5000,65000],
ytickvals=range(10,61,10),
yrange_err=[-10,10],
ytickvals_err=range(-5,6,5),
save=save_path)
plot_energy(plt_df,
region='TAS1',
color_idx=5,
yrange=[5000,65000],
ytickvals=range(10,61,10),
yrange_err=[-15,10],
ytickvals_err=range(-15,11,5),
save=save_path)
plot_energy(plt_df,
region='VIC1',
color_idx=6,
yrange=[65000,195000],
ytickvals=range(70,191,20),
yrange_err=[-10,10],
ytickvals_err=range(-10,11,5),
save=save_path)
Total Emissions#
plot_emissions(plt_df,
region='NEM',
color_idx=1,
yrange=[225000,575000],
ytickvals=range(250,561,50),
yrange_err=[-10,15],
ytickvals_err=range(-10,16,5),
save=save_path)
plot_emissions(plt_df,
region='NSW1',
color_idx=2,
yrange=[70000,210000],
ytickvals=range(80,201,20),
yrange_err=[-7,7],
ytickvals_err=range(-5,6,5),
save=save_path)
plot_emissions(plt_df,
region='QLD1',
color_idx=3,
yrange=[85000,175000],
ytickvals=range(90,171,10),
yrange_err=[-12,7],
ytickvals_err=range(-10,6,5),
save=save_path)
plot_emissions(plt_df,
region='SA1',
color_idx=4,
yrange=[-3000,33000],
ytickvals=range(0,31,5),
yrange_err=[-8,13],
ytickvals_err=range(-5,11,5),
save=save_path)
plot_emissions(plt_df,
region='TAS1',
color_idx=5,
yrange=[-300,7300],
ytickvals=range(0,8,1),
yrange_err=[-8,13],
ytickvals_err=range(-5,11,5),
save=save_path, err_off=True)
plot_emissions(plt_df,
region='VIC1',
color_idx=6,
yrange=[30000,210000],
ytickvals=range(40,201,20),
yrange_err=[-15,5],
ytickvals_err=range(-15,6,5),
save=save_path)
Emissions Intensity#
plot_intensity(plt_df,
region='NEM',
color_idx=1,
yrange=[0.45,1.05],
ytickvals=range(50,101,10),
yrange_err=[-12,7],
ytickvals_err=range(-10,6,5),
save=save_path)
plot_intensity(plt_df,
region='NSW1',
color_idx=2,
yrange=[0.45,1.05],
ytickvals=range(50,101,10),
yrange_err=[-12,2],
ytickvals_err=range(-10,1,5),
save=save_path)
plot_intensity(plt_df,
region='QLD1',
color_idx=3,
yrange=[0.55,1.05],
ytickvals=range(60,101,10),
yrange_err=[-15,5],
ytickvals_err=range(-15,6,5),
save=save_path)
plot_intensity(plt_df,
region='SA1',
color_idx=4,
yrange=[-0.05,0.95],
ytickvals=range(0,96,10),
yrange_err=[-7,12],
ytickvals_err=range(-5,11,5),
save=save_path)
plot_intensity(plt_df,
region='TAS1',
color_idx=5,
yrange=[-0.03,0.28],
ytickvals=range(0,26,5),
yrange_err=[-7,12],
ytickvals_err=range(-5,11,5),
save=save_path, err_off=True)
plot_intensity(plt_df,
region='VIC1',
color_idx=6,
yrange=[0.45,1.45],
ytickvals=range(50,141,10),
yrange_err=[-15,5],
ytickvals_err=range(-15,6,5),
save=save_path)