#
# Open Power System Data
#
# Time series Datapackage
#
# sources.yml : Parameters required for downloading and parsing data from sources
#
ENTSO-E Transparency FTP:
    Actual Generation per Production Type:
        address: '62.209.222.9'
        user: TP_export
        passwd: eG75pLwgyfyQLzjJ
        path: export/export/AggregatedGenerationPerType
        filename: '{u_start.year}_{u_start.month}_AggregatedGenerationPerType.csv'
        frequency: monthly
        start: 2014-12-01
        end: recent
        filetype: csv
        resolution:
            - 15min
            - 30min
            - 60min
        web: https://transparency.entsoe.eu/generation/r2/actualGenerationPerProductionType/show
        columns:
            - DateTime
            - AreaTypeCode
            - MapCode
            - ProductionType_Name
            - ActualConsumption
            - ActualGenerationOutput
    Actual Total Load:
        address: '62.209.222.9'
        user: TP_export
        passwd: eG75pLwgyfyQLzjJ
        path: export/export/ActualTotalLoad
        filename: '{u_start.year}_{u_start.month}_ActualTotalLoad.csv'
        frequency: monthly
        start: 2014-12-01
        end: recent
        filetype: csv
        resolution:
            - 15min
            - 30min
            - 60min
        web: https://transparency.entsoe.eu/load-domain/r2/totalLoadR2/show
        columns:
            - DateTime
            - AreaTypeCode
            - MapCode
            - TotalLoadValue 
    Day Ahead Prices:
        address: '62.209.222.9'
        user: TP_export
        passwd: eG75pLwgyfyQLzjJ
        path: export/export/DayAheadPrices
        filename: '{u_start.year}_{u_start.month}_DayAheadPrices.csv'
        frequency: monthly
        start: 2014-12-01
        end: recent
        filetype: csv
        resolution:
            - 60min
        web: https://transparency.entsoe.eu/transmission-domain/r2/dayAheadPrices/show
        columns:
            - DateTime
            - AreaTypeCode
            - MapCode
            - Currency_IsoCode
            - Price
OPSD:
    capacity:
        url_template: http://data.open-power-system-data.org/renewable_power_plants/2018-02-27/renewable_capacity_timeseries_DE.csv
        filename: renewable_capacity_timeseries_DE.csv
        url_params_template:
        frequency: complete
        start: 2005-01-01
        end: 2017-12-31
        filetype: csv
        resolution:
            - 15min
        web: http://data.open-power-system-data.org/renewable_power_plants/

ENTSO-E Data Portal:
    load:
        url_template: https://www.entsoe.eu/fileadmin/template/other/statistical_database/excel.php
        url_params_template:
            pid: '136'
            opt_period: '0'
            send: send
            opt_Response: '99'
            dataindx: '0'
            opt_Month: '{u_start.month}'
            opt_Year: '{u_start.year}'
        frequency: monthly
        start: 2006-01-01
        end: 2015-12-31
        filetype: xls
        resolution:
            - 60min
        web: https://www.entsoe.eu/data/data-portal/consumption/Pages/default.aspx  # Hourly load values of all countries for a specific month

ENTSO-E Power Statistics:  # Automatic download not implemented
    load:
        frequency: monthly
        start: 2016-01-01
        end: recent
        filetype: xlsx
        resolution:
            - 60min
        web: https://www.entsoe.eu/data/data-portal/consumption/Pages/default.aspx

50Hertz:
    wind generation_actual pre-offshore:
        url_template: http://ws.50hertz.com/web02/api/WindPowerActual/DownloadFile
        url_params_template:
            callback: '?'
            fileName: '{u_start:%Y}.csv'
        frequency: yearly
        start: 2005-01-01
        end: 2015-12-31
        filetype: csv
        resolution:
            - 15min
        web: http://www.50hertz.com/en/Grid-Data/Wind-power/Archive-Wind-power
    wind generation_forecast pre-offshore:
        url_template: http://ws.50hertz.com/web02/api/WindPowerForecast/DownloadFile
        url_params_template:
            callback: '?'
            fileName: '{u_start:%Y}.csv'
        frequency: yearly
        start: 2005-01-01
        end: 2015-12-31
        filetype: csv
        resolution:
            - 15min
        web: http://www.50hertz.com/en/Grid-Data/Wind-power/Archive-Wind-power
    wind generation_actual with-offshore:
        url_template: http://ws.50hertz.com/web02/api/WindPowerActual/DownloadFile
        url_params_template:
            callback: '?'
            fileName: '{u_start:%Y}.csv'
        frequency: yearly
        start: 2016-01-01
        end: recent
        filetype: csv
        resolution:
            - 15min
        web: http://www.50hertz.com/en/Grid-Data/Wind-power/Archive-Wind-power
    wind generation_forecast with-offshore:
        url_template: http://ws.50hertz.com/web02/api/WindPowerForecast/DownloadFile
        url_params_template:
            callback: '?'
            fileName: '{u_start:%Y}.csv'
        frequency: yearly
        start: 2016-01-01
        end: recent
        filetype: csv
        resolution:
            - 15min
        web: http://www.50hertz.com/en/Grid-Data/Wind-power/Archive-Wind-power
    solar generation_actual :
        url_template: http://ws.50hertz.com/web02/api/PhotovoltaicActual/DownloadFile
        url_params_template:
            callback: '?'
            fileName: '{u_start:%Y}.csv'
        frequency: yearly
        start: 2012-01-01
        end: recent
        filetype: csv
        resolution:
            - 15min
        web: http://www.50hertz.com/en/Grid-Data/Photovoltaics/Archive-Photovoltaics
    solar generation_forecast:
        url_template: http://ws.50hertz.com/web02/api/PhotovoltaicForecast/DownloadFile
        url_params_template:
            callback: '?'
            fileName: '{u_start:%Y}.csv'
        frequency: yearly
        start: 2012-01-01
        end: recent
        filetype: csv
        resolution:
            - 15min
        web: http://www.50hertz.com/en/Grid-Data/Photovoltaics/Archive-Photovoltaics

Amprion:
    wind:
        url_template: http://amprion.de/applications/applicationfiles/winddaten2.php
        url_params_template:
            mode: download
            format: csv
            start: '{u_start.day}.{u_start.month}.{u_start.year}'
            end: '{u_end.day}.{u_end.month}.{u_end.year}' # dates must not be zero-padded
        frequency: complete
        start: 2008-01-04
        end: recent
        filetype: csv
        resolution:
            - 15min
        web: http://www.amprion.net/en/wind-feed-in
    solar:
        url_template: http://amprion.de/applications/applicationfiles/PV_einspeisung.php
        url_params_template:
            mode: download
            format: csv
            start: '{u_start.day}.{u_start.month}.{u_start.year}'
            end: '{u_end.day}.{u_end.month}.{u_end.year}' # dates must not be zero-padded
        frequency: complete
        start: 2010-01-07
        end: recent
        filetype: csv
        resolution:
            - 15min
        web: http://www.amprion.net/en/photovoltaic-infeed

TenneT:
    wind:
        url_template: http://www.tennettso.de/site/de/phpbridge
        url_params_template:
            commandpath: Tatsaechliche_und_prognostizierte_Windenergieeinspeisung/monthDataSheetCsv.php
            contenttype: text/x-csv
            querystring: monat={u_start:%Y-%m}
        frequency: monthly
        start: 2006-01-01
        end: recent
        filetype: csv
        resolution:
            - 15min
        web: http://www.tennettso.de/site/en/Transparency/publications/network-figures/actual-and-forecast-wind-energy-feed-in
    solar:
        url_template: http://www.tennettso.de/site/de/phpbridge
        url_params_template:
            commandpath: Tatsaechliche_und_prognostizierte_Solarenergieeinspeisung/monthDataSheetCsv.php
            sub: total
            contenttype: text/x-csv
            querystring: monat={u_start:%Y-%m}
        frequency: monthly
        start: 2010-01-01
        end: recent
        filetype: csv
        resolution:
            - 15min
        web: http://www.tennettso.de/site/en/Transparency/publications/network-figures/actual-and-forecast-photovoltaic-energy-feed-in

TransnetBW:
    wind:
        url_template: https://api.transnetbw.de/wind/csv
        url_params_template:
            date: '{u_start:%Y-%m}'
        frequency: monthly
        start: 2010-01-01
        end: recent
        filetype: csv
        resolution:
            - 15min
        web: https://www.transnetbw.com/en/transparency/market-data/key-figures
    solar:
        url_template: https://api.transnetbw.de/photovoltaics/csv
        url_params_template:
            date: '{u_start:%Y-%m}'
        frequency: monthly
        start: 2011-01-01
        end: recent
        filetype: csv
        resolution:
            - 15min
        web: https://www.transnetbw.com/en/transparency/market-data/key-figures

Svenska Kraftnaet:
    wind_solar_1:
        url_template: http://www.svk.se/siteassets/aktorsportalen/elmarknad/statistik/sverigestatistik/n_fot{u_start:%Y}.xls
        url_params_template:
        frequency: yearly
        start: 2002-01-01
        end: 2009-12-31
        filetype: xls
        resolution:
            - 60min
        web: http://www.svk.se/aktorsportalen/elmarknad/statistik/
        filename: n_fot{u_start:%Y}.xls
    wind_solar_2:
        url_template: http://www.svk.se/siteassets/aktorsportalen/elmarknad/statistik/sverigestatistik/n_fot201001-06.xls
        url_params_template:
        frequency: irregular
        start: 2010-01-01
        end: 2010-06-30
        filetype: xls
        resolution:
            - 60min
        web: http://www.svk.se/aktorsportalen/elmarknad/statistik/
        filename: n_fot201001-06.xls
    wind_solar_3:
        url_template: http://www.svk.se/siteassets/aktorsportalen/elmarknad/statistik/sverigestatistik/n_fot2010-07-12.xls
        url_params_template:
        frequency: irregular
        start: 2010-07-01
        end: 2010-12-31
        filetype: xls
        resolution:
            - 60min
        web: http://www.svk.se/aktorsportalen/elmarknad/statistik/
        filename: n_fot2010-07-12.xls
    wind_solar_4:
        url_template: http://www.svk.se/siteassets/aktorsportalen/elmarknad/statistik/sverigestatistik/n-fot2011-01-12.xls
        url_params_template:
        frequency: yearly
        start: 2011-01-01
        end: 2011-12-31
        filetype: xls
        resolution:
            - 60min
        web: http://www.svk.se/aktorsportalen/elmarknad/statistik/
        filename: n-fot2011-01-12.xls
    wind_solar_5:
        url_template: http://www.svk.se/siteassets/aktorsportalen/elmarknad/statistik/sverigestatistik/n_fot{u_start:%Y}-01-12.xls
        url_params_template:
        frequency: yearly
        start: 2012-01-01
        end: 2014-12-31
        filetype: xls
        resolution:
            - 60min
        web: http://www.svk.se/aktorsportalen/elmarknad/statistik/
        filename: n_fot{u_start:%Y}-01-12.xls
    wind_solar_6:
        url_template: http://www.svk.se/siteassets/aktorsportalen/elmarknad/statistik/sverigestatistik/forbrukning-och-tillforsel-per-timme-i-normaltid-{u_start:%Y}.xls
        url_params_template:
        frequency: yearly
        start: 2015-01-01
        end: 2017-12-31
        filetype: xls
        resolution:
            - 60min
        web: http://www.svk.se/aktorsportalen/elmarknad/statistik/
        filename: forbrukning-och-tillforsel-per-timme-i-normaltid-{u_start:%Y}.xls

Elia:
    wind-onshore:
        url_template: http://publications.elia.be/Publications/Publications/WindForecasting.v2.svc/ExportForecastData
        url_params_template:
            beginDate: '{u_start:%Y-%m-%dT%H:%M:%S.000Z}'
            endDate: '{u_end:%Y-%m-%dT%H:%M:%S.000Z}'
            isOffshore: 'false'
            isEliaConnected: ''
        frequency: monthly #the frequency could in principle be chosen arbitrarily, even complete the complete dataset, but in practise the server often times out if too much data is requested at once
        start: 2012-01-19  #the data starts from 2012-01-19
        end: recent
        filetype: xls
        resolution:
            - 15min
        web: http://www.elia.be/en/grid-data/power-generation/wind-power
    wind-offshore:
        url_template: http://publications.elia.be/Publications/Publications/WindForecasting.v2.svc/ExportForecastData
        url_params_template:
            beginDate: '{u_start:%Y-%m-%dT%H:%M:%S.000Z}'
            endDate: '{u_end:%Y-%m-%dT%H:%M:%S.000Z}'
            isOffshore: 'true'
            isEliaConnected: ''
        frequency: quarterly
        start: 2012-01-19  #the data starts from 2012-01-19
        end: recent
        filetype: xls
        resolution:
            - 15min
        web: http://www.elia.be/en/grid-data/power-generation/wind-power
    solar:
        url_template: http://publications.elia.be/Publications/Publications/SolarForecasting.v3.svc/ExportSolarForecastGraph
        url_params_template:
            dateFrom: '{u_start:%Y-%m-%d}T23:00:00.000Z'
            dateTo: '{u_end:%Y-%m-%d}T23:00:00.000Z'
            sourceId: '1'
        frequency: monthly
        start: 2012-11-14 #the data starts from 2012-11-14
        end: recent
        filetype: xls
        resolution:
            - 15min
        web: http://www.elia.be/en/grid-data/power-generation/Solar-power-generation-data/Graph

Energinet.dk:  # Automatic download not implemented
    prices_wind_solar:
        frequency: complete
        start: 2000-01-01 # or even earlier for some data, but not useful as other sources don't go as far back
        end: recent
        filetype: xls
        resolution:
            - 60min
        web: http://osp.energinet.dk/_layouts/Markedsdata/framework/integrations/markedsdatatemplate.aspx
#     prices_2:
#         url_template: https://stapckanedsfiles01.blob.core.windows.net/edsprod/Elspot_Prices_Data-d2f38eb6e3a74cdab724fec1c050689f.csv
#         frequency: complete
#         start: 2016  # 100000 rows download limit
#         end: recent
#         filetype: csv
#         resolution:
#             - 60min
#         web: https://www.energidataservice.dk/dataset/elspotprices/
#     wind_solar_2:
#         url_template: https://stapckanedsfiles01.blob.core.windows.net/edsprod/Electricity_Balance_Data-88d6261333094e56a30f8f8a94ffe509.csv
#         frequency: complete
#         start: 2012  # 100000 rows download limit
#         end: recent
#         filetype: csv
#         resolution:
#             - 60min
#         web: https://www.energidataservice.dk/dataset/electricitybalance/

CEPS:  # Automatic download not implemented
    wind_pv:
        frequency: complete
        start: 2012-01-01
        end: recent
        filetype: xls
        resolution:
            - 60min
        web: http://www.ceps.cz/en/all-data#GenerationRES

PSE:
    wind:
        url_template: https://www.pse.pl/getcsv/-/export/csv/Operation_PPS_G_WF/data/{u_start:%Y%m%d}
        url_params_template:
        frequency: daily
        start:  2012-05-26
        end: recent
        filetype: csv
        resolution:
            - 60min
        web: https://www.pse.pl/web/pse-eng/data/polish-power-system-operation/generation-in-wind-farms

APG:
    generation_by_source:
        url_template: https://www.apg.at/transparency/Download.aspx?
        url_params_template:
            LANGUAGE: de
            PID: AGPT
            Format: csv
            Resolution: 15M
            DateFromLocal: '{u_start:%Y%m%d}'
            DateToLocal: '{u_end:%Y%m%d}'
        #    CacheDir: 20170428131612_4266553c941447f4a8836f88af6cc4ef  #is unnecessary
            CacheFileName: export_agpt_{u_start:%Y-%m-%d}T00_00_00Z_{u_end:%Y-%m-%d}T23_45_00Z_15M_de.csv
        frequency: quarterly
        start: 2006-01-01
        end: recent
        filetype: csv
        resolution:
            - 15min
        web: https://www.apg.at/de/markt/Markttransparenz/erzeugung/Erzeugung pro Typ

RTE:
    generation_by_source:
        url_template: http://clients.rte-france.com/servlets/RealProdServlet
        url_params_template:
            annee: '{u_start:%Y}'
        frequency: yearly
        start: 2014-01-01 # real start date is 2014-12-16. download.py only works with beginning of year though.
        end: recent
        filetype: zip #xls
        resolution:
            - 60min
        web: http://clients.rte-france.com/lang/an/visiteurs/vie/prod/realisation_production.jsp
#
# For the Following sources, the download/read has not yet been implemented
#
#Fingrid:
#    wind:
#        url_template: http://www.fingrid.fi/en/electricity-market/load-and-generation/windpower/Pages/default.aspx/Excel/TimeSeries.xls?
#        url_params_template:
#            beginDate: '{u_start:%Y%m%d}'
#            endDate: '{u_start:%Y%m%d}'
#            variables: H4sIAAAAAAAEAO29B2AcSZYlJi9tynt_SvVK1-B0oQiAYBMk2JBAEOzBiM3mkuwdaUcjKasqgcplVmVdZhZAzO2dvPfee--999577733ujudTif33_8_XGZkAWz2zkrayZ4hgKrIHz9-fB8_Ih7_ZFYX2aTMmyP769nyvDp6_CJb5EfPzn7_N1999fzsJ788--L49_99vv3m8V3-_PHZ8jKv25_MynV-dJ6VTf74rv_RY_7xrKoXWfu6rYvlxdGPj378x3ce3-1_8firZdG-yd-1R198d353_viu_fvxm6It86PvFstZuqqu8jq9yJd5nbVFtUy3uPWdx3el0ePnxTI_qcqqTu96f3y7XZRHP_7k5MnDHeo8_Pjx63l19eWyvH62btd1_jRrMzOWyDdA3aeP_bM5-n8AhQmq0kUBAAA1
#            cultureId: en-US
#            dataTimePrecision: 5
#        frequency: monthly # ???
#        start: 2014-11-28
#        end: recent
#        filetype: xls
#        resolution:
#            - 60min
#        web:
#
#OeMag:
#    wind_1:
#        url_template: http://www.oem-ag.at/fileadmin/user_upload/Dokumente/statistik/winderzeugung/winderzeugung_{u_start:%Y}.xls
#        bundle: YEARLY
#        start: 2003-01-01
#        end: 2011-01-01
#        filetype: xls
#        resolution:
#            - 15min
#    wind_2:
#        url_template: http://www.oem-ag.at/fileadmin/user_upload/Dokumente/statistik/winderzeugung/winderzeugung_{u_start:%Y}.xlsx
#        bundle: YEARLY
#        start: 2012-01-01
#        end: 2013-12-31
#        filetype: xlsx
#        resolution:
#            - 15min
#    wind_3:
#        url_template: http://www.oem-ag.at/fileadmin/user_upload/Dokumente/statistik/winderzeugung/Winderzeugung_{u_start:%Y}.xlsx
#        bundle: YEARLY
#        start: 2013-01-01
#        end: 2013-12-31
#        filetype: xlsx
#        resolution:
#            - 15min
#
#Energinet.dk:  # Automatic download not implemented
#     prices_2:
#         url_template: https://stapckanedsfiles01.blob.core.windows.net/edsprod/Elspot_Prices_Data-d2f38eb6e3a74cdab724fec1c050689f.csv
#         frequency: complete
#         start: 2016  # 100000 rows download limit
#         end: recent
#         filetype: csv
#         resolution:
#             - 60min
#         web: https://www.energidataservice.dk/dataset/elspotprices/
#     wind_solar_2:
#         url_template: https://stapckanedsfiles01.blob.core.windows.net/edsprod/Electricity_Balance_Data-88d6261333094e56a30f8f8a94ffe509.csv
#         frequency: complete
#         start: 2012  # 100000 rows download limit
#         end: recent
#         filetype: csv
#         resolution:
#             - 60min
#         web: https://www.energidataservice.dk/dataset/electricitybalance/
#
#TEMPLATE
#
#source:
#    variable:
#        url_template:
#        url_params_template:
#
#        frequency:
#        start:
#        end: recent
#        filetype:
#        resolution:
#        web: