From a003c8ff812ddcc118ec62c5ad28313b13784e58 Mon Sep 17 00:00:00 2001 From: shubh Date: Fri, 18 Oct 2024 04:06:28 +0200 Subject: [PATCH 1/8] updated readme --- README.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index acf251f..8af85d1 100644 --- a/README.md +++ b/README.md @@ -1 +1,19 @@ -The core business logic of codegreen. Please see the documentation website for more details about the project. \ No newline at end of file +This repository contains the main functionality of the codegreen project. The complete documentation including installation and usage are available on the [documentation website](https://codegreen-framework.github.io/codegreen-core/). + +## Development workflow +- the `release` branch contains the latest stable version of the released python package +- the `main` branch contains stable, tested code ready to be released. +- the `dev` branch is the main working branch. All feature branches are merged to `dev` + +### Releasing the package +- Ensure new changes are merged to `dev` and the version number is bumped according to [sematic versioning](https://semver.org/). +- Merge `dev`to `main`. This should trigger tests. +- Create a git tag for the new version : +``` +git checkout main +git pull +git tag vX.Y.Z # Replace X.Y.Z with the new version number +git push origin vX.Y.Z +``` +- Create a PR from `main` to `release` and get one approval +- Once the PR is merged, it will trigger the release of the new package version and publish the documentation. \ No newline at end of file From ff3da958a20b01b24124796b7c2dc4b08d498076 Mon Sep 17 00:00:00 2001 From: shubh Date: Fri, 18 Oct 2024 04:21:30 +0200 Subject: [PATCH 2/8] fixed testing and added a workflow --- .github/workflows/test.yml | 43 +++++++++++++++++++ docs/models.rst | 20 --------- ...st_predictions.py => test1_predictions.py} | 1 + tests/test_data.py | 5 ++- tests/test_loadshift_location.py | 2 +- tests/test_loadshift_time.py | 25 ++++++----- 6 files changed, 60 insertions(+), 36 deletions(-) create mode 100644 .github/workflows/test.yml delete mode 100644 docs/models.rst rename tests/{test_predictions.py => test1_predictions.py} (88%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..7dbee46 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,43 @@ +name: Run Tests + +# Trigger this workflow when pushing to the main branch +on: + push: + branches: + - main + pull_request: + branches: + - main + +# Define the job to run the tests +jobs: + test: + runs-on: ubuntu-latest + + steps: + # Checkout the code from the repository + - name: Checkout repository + uses: actions/checkout@v3 + + # Set up Python environment + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.12.5' # Specify Python version (e.g., '3.9') + + # Install dependencies (you can specify requirements.txt or pyproject.toml) + - name: Install dependencies and move to the test folder + run: | + python -m pip install --upgrade pip + pip install . + cd tests + + # Run pytest to execute tests + - name: Generate .config file inside the test folder + run: | + echo "[codegreen]" > .codegreencore.config + echo "ENTSOE_token=${{ secrets.ENTSOE_TOKEN }}" >> .codegreencore.config + echo "enable_energy_caching=false" >> .codegreencore.config + - name: Run tests + run: | + pytest diff --git a/docs/models.rst b/docs/models.rst deleted file mode 100644 index 1011bf4..0000000 --- a/docs/models.rst +++ /dev/null @@ -1,20 +0,0 @@ -Models Module -============= - -This section covers the `codegreen_core.models` module. - -Main Functions --------------- - -.. automodule:: codegreen_core.models - :members: - :undoc-members: - :show-inheritance: - -Submodules ----------- - -.. automodule:: codegreen_core.models.predict - :members: - :undoc-members: - :show-inheritance: diff --git a/tests/test_predictions.py b/tests/test1_predictions.py similarity index 88% rename from tests/test_predictions.py rename to tests/test1_predictions.py index 62ee9e0..403f51f 100644 --- a/tests/test_predictions.py +++ b/tests/test1_predictions.py @@ -1,3 +1,4 @@ +# this code is not yet used from codegreen_core.models import predict from codegreen_core.data import energy from datetime import datetime diff --git a/tests/test_data.py b/tests/test_data.py index a5ae213..cc1c9e7 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -1,5 +1,6 @@ import pytest -from codegreen_core.data import entsoe,energy,carbon_intensity +from codegreen_core.data import energy +from codegreen_core.data.entsoe import renewableSources from codegreen_core.utilities.message import CodegreenDataError from datetime import datetime import pandas as pd @@ -84,7 +85,7 @@ def test_entsoe_generation_data(self): end_utc = pd.to_datetime(case["end"]) #case["end"].astimezone(pd.Timestamp.now(tz='UTC').tzinfo) if case["end"].tzinfo is None else case["end"] filtered_df = data_verify[(data_verify['start_date'] >= start_utc) & (data_verify['start_date'] < end_utc)] allCols = data.columns.tolist() - renPresent = list(set(allCols).intersection(entsoe.renewableSources)) + renPresent = list(set(allCols).intersection(renewableSources)) for e in renPresent: difference = filtered_df[e+" - Actual Aggregated [MW]"] - data[e] sum_of_differences = difference.sum() diff --git a/tests/test_loadshift_location.py b/tests/test_loadshift_location.py index 04974ad..f4ebfea 100644 --- a/tests/test_loadshift_location.py +++ b/tests/test_loadshift_location.py @@ -40,4 +40,4 @@ def test_locations(): a,b,c,d = predict_optimal_location(data,case["h"],case["m"],case["p"],end,start) print(a,b,c,d) -test_locations() \ No newline at end of file +# test_locations() \ No newline at end of file diff --git a/tests/test_loadshift_time.py b/tests/test_loadshift_time.py index 773dce1..bf20ef2 100644 --- a/tests/test_loadshift_time.py +++ b/tests/test_loadshift_time.py @@ -1,4 +1,3 @@ -import codegreen_core.tools as tools import pytest from codegreen_core.utilities.message import CodegreenDataError,Message from datetime import datetime,timezone,timedelta @@ -179,17 +178,17 @@ def test_data_validation_country(self): print(timestamp1,timestamp, message) assert timestamp - timestamp1 <= 10 assert message == Message.ENERGY_DATA_FETCHING_ERROR - def test_all_country_test(self): - test_cases = pd.read_csv("./data/test_cases_time.csv") - data = pd.read_csv("./data/prediction_testing_data.csv") - for index, row in test_cases.iterrows(): - edata_filter = data["file_id"] == row["country"] - energy_data = data[edata_filter].copy() - start = datetime.strptime(row["start_time"], '%Y-%m-%d %H:%M:%S') - end = (start + timedelta(hours=row["hard_deadline_hour"])) - a,b,c = ts.predict_optimal_time(energy_data,row["runtime_hour"],row["runtime_min"],row["percent_renewable"],end,start) - print(a,b,c) - assert int(a) == row["expected_timestamp"] + # def test_all_country_test(self): + # test_cases = pd.read_csv("./data/test_cases_time.csv") + # data = pd.read_csv("./data/prediction_testing_data.csv") + # for index, row in test_cases.iterrows(): + # edata_filter = data["file_id"] == row["country"] + # energy_data = data[edata_filter].copy() + # start = datetime.strptime(row["start_time"], '%Y-%m-%d %H:%M:%S') + # end = (start + timedelta(hours=row["hard_deadline_hour"])) + # a,b,c = ts.predict_optimal_time(energy_data,row["runtime_hour"],row["runtime_min"],row["percent_renewable"],end,start) + # print(a,b,c) + # assert int(a) == row["expected_timestamp"] # for case in cases: # #print(case) @@ -218,7 +217,7 @@ def test_all_country(): assert int(a) == row["expected_timestamp"] print("====") -test_all_country() +# test_all_country() # def data_validation_country(): From 473678073cfa3f3ac2dc37b3b33f35a1196957b3 Mon Sep 17 00:00:00 2001 From: shubh Date: Fri, 18 Oct 2024 04:22:32 +0200 Subject: [PATCH 3/8] ... --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7dbee46..3cb29e5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,7 +34,7 @@ jobs: # Run pytest to execute tests - name: Generate .config file inside the test folder - run: | + run: | echo "[codegreen]" > .codegreencore.config echo "ENTSOE_token=${{ secrets.ENTSOE_TOKEN }}" >> .codegreencore.config echo "enable_energy_caching=false" >> .codegreencore.config From 959e20aee88f5b65039d7eee0b0c65e3b50ea6a9 Mon Sep 17 00:00:00 2001 From: shubh Date: Fri, 18 Oct 2024 04:26:04 +0200 Subject: [PATCH 4/8] added install cmd --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3cb29e5..4ac06df 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,6 +30,7 @@ jobs: run: | python -m pip install --upgrade pip pip install . + pip install pytest cd tests # Run pytest to execute tests From 8bbac0f17ba0e35e512ede34b187b0e436f7d29a Mon Sep 17 00:00:00 2001 From: shubh Date: Fri, 18 Oct 2024 04:46:38 +0200 Subject: [PATCH 5/8] fixing test pipeline --- .github/workflows/test.yml | 13 +++++++------ README.md | 2 ++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4ac06df..b35871d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,4 @@ -name: Run Tests +name: Run Tests on main # Trigger this workflow when pushing to the main branch on: @@ -29,16 +29,17 @@ jobs: - name: Install dependencies and move to the test folder run: | python -m pip install --upgrade pip + pip install pytest pandas numpy entsoe-py redis tensorflow scikit-learn sphinx pip install . - pip install pytest - cd tests + # Run pytest to execute tests - name: Generate .config file inside the test folder run: | - echo "[codegreen]" > .codegreencore.config - echo "ENTSOE_token=${{ secrets.ENTSOE_TOKEN }}" >> .codegreencore.config - echo "enable_energy_caching=false" >> .codegreencore.config + echo "[codegreen]" > tests/.codegreencore.config + echo "ENTSOE_token=${{ secrets.ENTSOE_TOKEN }}" >> tests/.codegreencore.config + echo "enable_energy_caching=false" >> tests/.codegreencore.config - name: Run tests run: | + cd ./tests pytest diff --git a/README.md b/README.md index 8af85d1..4a2d457 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![Run Tests](https://github.com/codegreen-framework/codegreen-core/actions/workflows/test.yml/badge.svg)](https://github.com/codegreen-framework/codegreen-core/actions/workflows/test.yml) + This repository contains the main functionality of the codegreen project. The complete documentation including installation and usage are available on the [documentation website](https://codegreen-framework.github.io/codegreen-core/). ## Development workflow From 6fd42f396eaf8f18bef3daba42e91729211c1293 Mon Sep 17 00:00:00 2001 From: shubh Date: Fri, 18 Oct 2024 11:19:01 +0200 Subject: [PATCH 6/8] fixed path of test files --- .github/workflows/test.yml | 2 -- tests/test_data.py | 8 ++++---- tests/test_loadshift_location.py | 2 +- tests/test_loadshift_time.py | 8 ++++---- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b35871d..38e266a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,7 +32,6 @@ jobs: pip install pytest pandas numpy entsoe-py redis tensorflow scikit-learn sphinx pip install . - # Run pytest to execute tests - name: Generate .config file inside the test folder run: | @@ -41,5 +40,4 @@ jobs: echo "enable_energy_caching=false" >> tests/.codegreencore.config - name: Run tests run: | - cd ./tests pytest diff --git a/tests/test_data.py b/tests/test_data.py index cc1c9e7..1cb6f35 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -33,7 +33,7 @@ def test_entsoe_generation_data(self): "start":datetime(2024,2,1), "end":datetime(2024,2,2), "dtype": 'generation' , - "file": "data/generation_DE_24_downloaded.csv", + "file": "tests/data/generation_DE_24_downloaded.csv", "interval60": False }, { @@ -41,7 +41,7 @@ def test_entsoe_generation_data(self): "start":datetime(2024,3,20), "end":datetime(2024,3,24), "dtype": 'generation' , - "file": "data/generation_DE_24_downloaded.csv", + "file": "tests/data/generation_DE_24_downloaded.csv", "interval60": False }, # { @@ -58,7 +58,7 @@ def test_entsoe_generation_data(self): "start":datetime(2024,3,20), "end":datetime(2024,3,24), "dtype": 'generation' , - "file": "data/generation_GR_24_downloaded.csv", + "file": "tests/data/generation_GR_24_downloaded.csv", "interval60": True }, { @@ -66,7 +66,7 @@ def test_entsoe_generation_data(self): "start":datetime(2024,1,25), "end":datetime(2024,1,28), "dtype": 'generation' , - "file": "data/generation_GR_24_downloaded.csv", + "file": "tests/data/generation_GR_24_downloaded.csv", "interval60": True } diff --git a/tests/test_loadshift_location.py b/tests/test_loadshift_location.py index f4ebfea..1c66a9c 100644 --- a/tests/test_loadshift_location.py +++ b/tests/test_loadshift_location.py @@ -10,7 +10,7 @@ def test_location_now(): # test_location_now() def fetch_data(month_no,countries): - data = pd.read_csv("data/prediction_testing_data.csv") + data = pd.read_csv("tests/data/prediction_testing_data.csv") forecast_data = {} for c in countries: filter = data["file_id"] == c+""+str(month_no) diff --git a/tests/test_loadshift_time.py b/tests/test_loadshift_time.py index bf20ef2..b959173 100644 --- a/tests/test_loadshift_time.py +++ b/tests/test_loadshift_time.py @@ -68,13 +68,13 @@ def test_less_energy_data(self): def test_if_incorrect_data_provided(self): """this is to test if energy data provided does not contain the data for the request time """ - data = pd.read_csv("data/DE_forecast1.csv") + data = pd.read_csv("tests/data/DE_forecast1.csv") timestamp, message, average_percent_renewable = ts.predict_optimal_time(data,20,0,10,self.hard_finish_time_2,self.request_time_2) assert timestamp == int(self.request_time_2.timestamp()) assert message == Message.NO_DATA def test_multiple(self): - data = pd.read_csv("data/DE_forecast1.csv") + data = pd.read_csv("tests/data/DE_forecast1.csv") hard_finish_time = datetime(2024,1,7,0,0) request_time = datetime(2024,1,5,0,0) cases = [ @@ -200,8 +200,8 @@ def test_data_validation_country(self): # test if request time is none current time is being used def test_all_country(): - test_cases = pd.read_csv("./data/test_cases_time.csv") - data = pd.read_csv("./data/prediction_testing_data.csv") + test_cases = pd.read_csv("tests/data/test_cases_time.csv") + data = pd.read_csv("tests/data/prediction_testing_data.csv") for _ , row in test_cases.iterrows(): print(row) edata_filter = data["file_id"] == row["country"] From e5436db2d3a4ee6a8d3020a72867f5bf7004a479 Mon Sep 17 00:00:00 2001 From: shubh Date: Fri, 18 Oct 2024 11:21:37 +0200 Subject: [PATCH 7/8] loc fix --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 38e266a..821c7ff 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,9 +35,9 @@ jobs: # Run pytest to execute tests - name: Generate .config file inside the test folder run: | - echo "[codegreen]" > tests/.codegreencore.config - echo "ENTSOE_token=${{ secrets.ENTSOE_TOKEN }}" >> tests/.codegreencore.config - echo "enable_energy_caching=false" >> tests/.codegreencore.config + echo "[codegreen]" > .codegreencore.config + echo "ENTSOE_token=${{ secrets.ENTSOE_TOKEN }}" >> .codegreencore.config + echo "enable_energy_caching=false" >> .codegreencore.config - name: Run tests run: | pytest From ec9bbaeb78643fc997bc03757e1106fef2ddf093 Mon Sep 17 00:00:00 2001 From: shubh Date: Fri, 18 Oct 2024 11:26:41 +0200 Subject: [PATCH 8/8] file fix --- .github/workflows/test.yml | 2 +- codegreen_core/utilities/log.py | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 codegreen_core/utilities/log.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 821c7ff..0cd7248 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,7 +26,7 @@ jobs: python-version: '3.12.5' # Specify Python version (e.g., '3.9') # Install dependencies (you can specify requirements.txt or pyproject.toml) - - name: Install dependencies and move to the test folder + - name: Install dependencies run: | python -m pip install --upgrade pip pip install pytest pandas numpy entsoe-py redis tensorflow scikit-learn sphinx diff --git a/codegreen_core/utilities/log.py b/codegreen_core/utilities/log.py new file mode 100644 index 0000000..795995c --- /dev/null +++ b/codegreen_core/utilities/log.py @@ -0,0 +1,24 @@ +# to log stuff + +from .config import Config +from datetime import datetime +import os +import csv + + +def time_prediction(data): + if Config.get("enable_time_prediction_logging")==True: + current_date = datetime.now() + file_name = f"{current_date.strftime('%B')}_{current_date.year}.csv" + file_location = os.path.join(Config.get("time_prediction_log_folder_path"), file_name) + file_exists = os.path.exists(file_location) + # Open the file in append mode + with open(file_location, mode='a', newline='') as file: + writer = csv.DictWriter(file, fieldnames=data.keys()) + # If the file doesn't exist, write the header + if not file_exists: + writer.writeheader() + # Append the data to the file + writer.writerow(data) + else: + print("Logging not enabled") \ No newline at end of file