From 62d58b234cedb07dee711d733760abb2beffa0c7 Mon Sep 17 00:00:00 2001 From: berkingurcan Date: Sun, 6 Oct 2024 10:54:24 +0700 Subject: [PATCH 01/10] feat revise controller scheduler --- github_tracker_bot/bot.py | 61 ++++++++++++++++++++++++--------------- tasks.py | 2 ++ 2 files changed, 39 insertions(+), 24 deletions(-) diff --git a/github_tracker_bot/bot.py b/github_tracker_bot/bot.py index 57e3bd1..7547078 100644 --- a/github_tracker_bot/bot.py +++ b/github_tracker_bot/bot.py @@ -1,10 +1,9 @@ +from contextlib import asynccontextmanager import sys import os sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) -import utils - import asyncio from datetime import datetime, timedelta, timezone @@ -30,7 +29,26 @@ from slowapi.middleware import SlowAPIMiddleware from slowapi.errors import RateLimitExceeded -app = FastAPI() + +@asynccontextmanager +async def lifespan(app: FastAPI): + app.state.scheduler_task = asyncio.create_task(scheduler()) + logger.info("Scheduler started on application startup") + + try: + yield + finally: + if app.state.scheduler_task: + app.state.scheduler_task.cancel() + try: + await app.state.scheduler_task + except asyncio.CancelledError: + pass + app.state.scheduler_task = None + logger.info("Scheduler stopped on application shutdown") + + +app = FastAPI(lifespan=lifespan) limiter = Limiter(key_func=get_remote_address, default_limits=["5/minute"]) @@ -38,12 +56,11 @@ app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler) app.add_middleware(SlowAPIMiddleware) -scheduler_task = None +app.state.scheduler_task = None class ScheduleControl(BaseModel): action: str - interval_minutes: int = 1 class TaskTimeFrame(BaseModel): @@ -80,8 +97,11 @@ async def run_scheduled_task(): raise -async def scheduler(interval_minutes): - schedule.every(interval_minutes).minutes.do(run_scheduled_task) +async def scheduler(): + async def job(): + await run_scheduled_task() + + schedule.every().day.at("00:02").do(lambda: asyncio.create_task(job())) while True: await schedule.run_pending() await asyncio.sleep(1) @@ -130,29 +150,22 @@ async def run_task_for_user( @app.post("/control-scheduler") async def control_scheduler(control: ScheduleControl): - global scheduler_task - if control.action == "start": - if scheduler_task is None or scheduler_task.cancelled(): - interval_minutes = ( - control.interval_minutes or 1 - ) # Default to 1 minute if not specified - scheduler_task = asyncio.create_task(scheduler(interval_minutes)) - return { - "message": "Scheduler started with interval of {} minutes".format( - interval_minutes - ) - } + if ( + app.state.scheduler_task is None + or app.state.scheduler_task.cancelled() + or app.state.scheduler_task.done() + ): + app.state.scheduler_task = asyncio.create_task(scheduler()) + return {"message": "Scheduler started"} else: return {"message": "Scheduler is already running"} - elif control.action == "stop": - if scheduler_task and not scheduler_task.cancelled(): - scheduler_task.cancel() - scheduler_task = None + if app.state.scheduler_task and not app.state.scheduler_task.cancelled(): + app.state.scheduler_task.cancel() + app.state.scheduler_task = None return {"message": "Scheduler stopped"} return {"message": "Scheduler is not running"} - else: raise HTTPException(status_code=400, detail="Invalid action specified") diff --git a/tasks.py b/tasks.py index 09670b1..6c82350 100644 --- a/tasks.py +++ b/tasks.py @@ -30,10 +30,12 @@ def testmongo(ctx): def testmongoint(ctx): ctx.run(f"python -m unittest tests/test_mongo_integration.py") + @task def testextract(ctx): ctx.run(f"python -m unittest tests/test_extract_unnecessary_diff.py") + @task def testss(ctx): ctx.run(f"python -m unittest tests/test_spreadsheet_to_list_of_user.py") From 39b60eb22ae835e912be1f6d1b5649ec59f4d684 Mon Sep 17 00:00:00 2001 From: berkingurcan Date: Sun, 6 Oct 2024 11:16:05 +0700 Subject: [PATCH 02/10] feat increase rate limit --- github_tracker_bot/bot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/github_tracker_bot/bot.py b/github_tracker_bot/bot.py index 7547078..0eefbc2 100644 --- a/github_tracker_bot/bot.py +++ b/github_tracker_bot/bot.py @@ -1,4 +1,3 @@ -from contextlib import asynccontextmanager import sys import os @@ -13,6 +12,7 @@ from pydantic import BaseModel, Field, field_validator import aioschedule as schedule +from contextlib import asynccontextmanager from github_tracker_bot.bot_functions import ( get_all_results_from_sheet_by_date, @@ -50,7 +50,7 @@ async def lifespan(app: FastAPI): app = FastAPI(lifespan=lifespan) -limiter = Limiter(key_func=get_remote_address, default_limits=["5/minute"]) +limiter = Limiter(key_func=get_remote_address, default_limits=["10/minute"]) app.state.limiter = limiter app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler) From 5c27152d79bb25e3bfd286d8d7d6d0166143627b Mon Sep 17 00:00:00 2001 From: berkingurcan Date: Sun, 6 Oct 2024 11:28:27 +0700 Subject: [PATCH 03/10] add reqs pytest asyncio --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 6d787dc..3ca876a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -76,6 +76,7 @@ pymongo==4.8.0 PyNaCl==1.5.0 pyparsing==3.1.2 pytest==8.2.2 +pytest-asyncio==0.24.0 python-dateutil==2.9.0.post0 python-dotenv==1.0.1 python-multipart==0.0.9 From c6c3fbfff54a695135e78fa29bc7b9a337fda396 Mon Sep 17 00:00:00 2001 From: berkingurcan Date: Sun, 6 Oct 2024 11:43:41 +0700 Subject: [PATCH 04/10] add logs and tests --- github_tracker_bot/bot.py | 2 + tests/test_bot_integration.py | 123 +++++++++++++++++++++++----------- 2 files changed, 86 insertions(+), 39 deletions(-) diff --git a/github_tracker_bot/bot.py b/github_tracker_bot/bot.py index 0eefbc2..9ac7053 100644 --- a/github_tracker_bot/bot.py +++ b/github_tracker_bot/bot.py @@ -157,6 +157,7 @@ async def control_scheduler(control: ScheduleControl): or app.state.scheduler_task.done() ): app.state.scheduler_task = asyncio.create_task(scheduler()) + logger.info(f"Scheduler started!") return {"message": "Scheduler started"} else: return {"message": "Scheduler is already running"} @@ -164,6 +165,7 @@ async def control_scheduler(control: ScheduleControl): if app.state.scheduler_task and not app.state.scheduler_task.cancelled(): app.state.scheduler_task.cancel() app.state.scheduler_task = None + logger.info(f"Scheduler stopped!") return {"message": "Scheduler stopped"} return {"message": "Scheduler is not running"} else: diff --git a/tests/test_bot_integration.py b/tests/test_bot_integration.py index 6726df3..a533512 100644 --- a/tests/test_bot_integration.py +++ b/tests/test_bot_integration.py @@ -1,4 +1,8 @@ import unittest +import pytest +from unittest.mock import AsyncMock, patch +from httpx import AsyncClient + from fastapi.testclient import TestClient import sys @@ -11,47 +15,88 @@ client = TestClient(bot.app) - +@pytest.mark.asyncio class TestIntegration(unittest.TestCase): - def test_run_task_endpoint(self): - response = client.post( - "/run-task", - json={ - "since": "2023-01-01T00:00:00+00:00", - "until": "2023-01-02T00:00:00+00:00", - }, - ) - self.assertEqual(response.status_code, 200) - self.assertIn( - "Task run successfully with provided times", response.json().get("message") - ) - - def test_run_task_with_incorrect_date(self): - response = client.post( - "/run-task", - json={ - "since": "2024-06-31T00:00:00+00:00", - "until": "2024-03-02T00:00:00+00:00", - }, - ) - self.assertEqual(response.status_code, 422) - self.assertRaises(ValueError) - - def test_control_scheduler_start_endpoint(self): - response = client.post( - "/control-scheduler", json={"action": "start", "interval_minutes": 5} - ) - self.assertEqual(response.status_code, 200) - self.assertIn( - "Scheduler started with interval of 5 minutes", - response.json().get("message"), - ) - - def test_control_scheduler_stop_endpoint(self): - response = client.post("/control-scheduler", json={"action": "stop"}) - self.assertEqual(response.status_code, 200) - self.assertIn("Scheduler is not running", response.json().get("message")) + @pytest.fixture + async def client(self): + async with AsyncClient(app=bot.app, base_url="http://test") as ac: + yield ac + + @pytest.mark.asyncio + @patch("github_tracker_bot.bot_functions.get_all_results_from_sheet_by_date", new_callable=AsyncMock) + async def test_run_task(mock_get_all_results): + async with AsyncClient(app=bot.app, base_url="http://test") as client: + response = await client.post( + "/run-task", + json={ + "since": "2023-10-01T00:00:00Z", + "until": "2023-10-02T00:00:00Z" + }, + headers={"Authorization": "your_auth_token"} + ) + assert response.status_code == 200 + assert response.json() == {"message": "Task run successfully with provided times"} + mock_get_all_results.assert_awaited_once() + + @pytest.mark.asyncio + @patch("github_tracker_bot.bot_functions.get_user_results_from_sheet_by_date", new_callable=AsyncMock) + async def test_run_task_for_user(mock_get_user_results): + async with AsyncClient(app=bot.app, base_url="http://test") as client: + response = await client.post( + "/run-task-for-user?username=testuser", + json={ + "since": "2023-10-01T00:00:00Z", + "until": "2023-10-02T00:00:00Z" + }, + headers={"Authorization": "your_auth_token"} + ) + assert response.status_code == 200 + assert response.json() == {"message": "Task run successfully with provided times"} + mock_get_user_results.assert_awaited_once() + + @pytest.mark.asyncio + async def test_unauthorized_access(self, client): + async with AsyncClient(app=bot.app, base_url="http://test") as client: + response = await client.post( + "/run-task", + json={ + "since": "2023-10-01T00:00:00Z", + "until": "2023-10-02T00:00:00Z" + } + # No Authorization header + ) + assert response.status_code == 401 + assert response.json() == {"message": "Unauthorized"} + + @pytest.mark.asyncio + async def test_rate_limiting(): + async with AsyncClient(app=bot.app, base_url="http://test") as client: + headers = {"Authorization": "your_auth_token"} + for _ in range(10): + response = await client.post( + "/run-task", + json={ + "since": "2023-10-01T00:00:00Z", + "until": "2023-10-02T00:00:00Z" + }, + headers=headers + ) + assert response.status_code == 200 + + # 11th request should be rate limited + response = await client.post( + "/run-task", + json={ + "since": "2023-10-01T00:00:00Z", + "until": "2023-10-02T00:00:00Z" + }, + headers=headers + ) + assert response.status_code == 429 + + + if __name__ == "__main__": From 2dbbbac5109052382412c9c29944a2af9d75b3e1 Mon Sep 17 00:00:00 2001 From: berkingurcan Date: Sun, 6 Oct 2024 12:15:11 +0700 Subject: [PATCH 05/10] feat add tests --- tests/test_bot_integration.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/tests/test_bot_integration.py b/tests/test_bot_integration.py index a533512..67569d8 100644 --- a/tests/test_bot_integration.py +++ b/tests/test_bot_integration.py @@ -15,14 +15,8 @@ client = TestClient(bot.app) -@pytest.mark.asyncio class TestIntegration(unittest.TestCase): - @pytest.fixture - async def client(self): - async with AsyncClient(app=bot.app, base_url="http://test") as ac: - yield ac - @pytest.mark.asyncio @patch("github_tracker_bot.bot_functions.get_all_results_from_sheet_by_date", new_callable=AsyncMock) async def test_run_task(mock_get_all_results): @@ -56,7 +50,7 @@ async def test_run_task_for_user(mock_get_user_results): mock_get_user_results.assert_awaited_once() @pytest.mark.asyncio - async def test_unauthorized_access(self, client): + async def test_unauthorized_access(self): async with AsyncClient(app=bot.app, base_url="http://test") as client: response = await client.post( "/run-task", @@ -70,7 +64,7 @@ async def test_unauthorized_access(self, client): assert response.json() == {"message": "Unauthorized"} @pytest.mark.asyncio - async def test_rate_limiting(): + async def test_rate_limiting(self): async with AsyncClient(app=bot.app, base_url="http://test") as client: headers = {"Authorization": "your_auth_token"} for _ in range(10): @@ -96,8 +90,5 @@ async def test_rate_limiting(): assert response.status_code == 429 - - - if __name__ == "__main__": unittest.main() From d32b78885a266dc3305ff817b3ad9d99dab08ee4 Mon Sep 17 00:00:00 2001 From: berkingurcan Date: Sun, 6 Oct 2024 12:17:46 +0700 Subject: [PATCH 06/10] feat make changable scheduler time --- github_tracker_bot/bot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/github_tracker_bot/bot.py b/github_tracker_bot/bot.py index 9ac7053..7b2ebaf 100644 --- a/github_tracker_bot/bot.py +++ b/github_tracker_bot/bot.py @@ -97,11 +97,11 @@ async def run_scheduled_task(): raise -async def scheduler(): +async def scheduler(scheduled_time="00:02"): async def job(): await run_scheduled_task() - schedule.every().day.at("00:02").do(lambda: asyncio.create_task(job())) + schedule.every().day.at(scheduled_time).do(lambda: asyncio.create_task(job())) while True: await schedule.run_pending() await asyncio.sleep(1) From eae4aba7efdb87154e68dd6562bf52e0d9693f70 Mon Sep 17 00:00:00 2001 From: berkingurcan Date: Sun, 6 Oct 2024 13:01:21 +0700 Subject: [PATCH 07/10] fix scheduler date issue and tests --- github_tracker_bot/bot.py | 4 +-- tests/test_bot_integration.py | 49 +++++++++++++++++------------------ 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/github_tracker_bot/bot.py b/github_tracker_bot/bot.py index 7b2ebaf..b24a1eb 100644 --- a/github_tracker_bot/bot.py +++ b/github_tracker_bot/bot.py @@ -80,8 +80,8 @@ def validate_datetime(cls, value): def get_dates_for_today(): today = datetime.now(timezone.utc) - since_date = today.replace(hour=0, minute=0, second=0, microsecond=0) - until_date = since_date + timedelta(days=1) + until_date = today.replace(hour=0, minute=0, second=0, microsecond=0) + since_date = until_date - timedelta(days=1) return since_date.isoformat(), until_date.isoformat() diff --git a/tests/test_bot_integration.py b/tests/test_bot_integration.py index 67569d8..d8b9d5e 100644 --- a/tests/test_bot_integration.py +++ b/tests/test_bot_integration.py @@ -15,38 +15,43 @@ client = TestClient(bot.app) + class TestIntegration(unittest.TestCase): @pytest.mark.asyncio - @patch("github_tracker_bot.bot_functions.get_all_results_from_sheet_by_date", new_callable=AsyncMock) + @patch( + "github_tracker_bot.bot_functions.get_all_results_from_sheet_by_date", + new_callable=AsyncMock, + ) async def test_run_task(mock_get_all_results): async with AsyncClient(app=bot.app, base_url="http://test") as client: response = await client.post( "/run-task", - json={ - "since": "2023-10-01T00:00:00Z", - "until": "2023-10-02T00:00:00Z" - }, - headers={"Authorization": "your_auth_token"} + json={"since": "2023-10-01T00:00:00Z", "until": "2023-10-02T00:00:00Z"}, + headers={"Authorization": "your_auth_token"}, ) assert response.status_code == 200 - assert response.json() == {"message": "Task run successfully with provided times"} + assert response.json() == { + "message": "Task run successfully with provided times" + } mock_get_all_results.assert_awaited_once() @pytest.mark.asyncio - @patch("github_tracker_bot.bot_functions.get_user_results_from_sheet_by_date", new_callable=AsyncMock) + @patch( + "github_tracker_bot.bot_functions.get_user_results_from_sheet_by_date", + new_callable=AsyncMock, + ) async def test_run_task_for_user(mock_get_user_results): async with AsyncClient(app=bot.app, base_url="http://test") as client: response = await client.post( "/run-task-for-user?username=testuser", - json={ - "since": "2023-10-01T00:00:00Z", - "until": "2023-10-02T00:00:00Z" - }, - headers={"Authorization": "your_auth_token"} + json={"since": "2023-10-01T00:00:00Z", "until": "2023-10-02T00:00:00Z"}, + headers={"Authorization": "your_auth_token"}, ) assert response.status_code == 200 - assert response.json() == {"message": "Task run successfully with provided times"} + assert response.json() == { + "message": "Task run successfully with provided times" + } mock_get_user_results.assert_awaited_once() @pytest.mark.asyncio @@ -54,10 +59,7 @@ async def test_unauthorized_access(self): async with AsyncClient(app=bot.app, base_url="http://test") as client: response = await client.post( "/run-task", - json={ - "since": "2023-10-01T00:00:00Z", - "until": "2023-10-02T00:00:00Z" - } + json={"since": "2023-10-01T00:00:00Z", "until": "2023-10-02T00:00:00Z"}, # No Authorization header ) assert response.status_code == 401 @@ -72,20 +74,17 @@ async def test_rate_limiting(self): "/run-task", json={ "since": "2023-10-01T00:00:00Z", - "until": "2023-10-02T00:00:00Z" + "until": "2023-10-02T00:00:00Z", }, - headers=headers + headers=headers, ) assert response.status_code == 200 # 11th request should be rate limited response = await client.post( "/run-task", - json={ - "since": "2023-10-01T00:00:00Z", - "until": "2023-10-02T00:00:00Z" - }, - headers=headers + json={"since": "2023-10-01T00:00:00Z", "until": "2023-10-02T00:00:00Z"}, + headers=headers, ) assert response.status_code == 429 From 1ee1c15fd7085cfd9ecc7214506e58320d48b7c1 Mon Sep 17 00:00:00 2001 From: berkingurcan Date: Sun, 6 Oct 2024 13:02:34 +0700 Subject: [PATCH 08/10] add logs --- github_tracker_bot/bot.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/github_tracker_bot/bot.py b/github_tracker_bot/bot.py index b24a1eb..e172015 100644 --- a/github_tracker_bot/bot.py +++ b/github_tracker_bot/bot.py @@ -92,6 +92,8 @@ async def run_scheduled_task(): await get_all_results_from_sheet_by_date( config.SPREADSHEET_ID, since_date, until_date ) + logger.info(f"Gotten results between {since_date} and {until_date}") + except Exception as e: logger.error(f"An error occurred while running the scheduled task: {e}") raise From e5fc53ed62fc570b544a3639c7ad358e28f1d913 Mon Sep 17 00:00:00 2001 From: berkingurcan Date: Sun, 6 Oct 2024 13:11:05 +0700 Subject: [PATCH 09/10] Update bot.py --- github_tracker_bot/bot.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/github_tracker_bot/bot.py b/github_tracker_bot/bot.py index e172015..c66a7aa 100644 --- a/github_tracker_bot/bot.py +++ b/github_tracker_bot/bot.py @@ -103,7 +103,12 @@ async def scheduler(scheduled_time="00:02"): async def job(): await run_scheduled_task() + logger.info(f"Scheduler is set to run the task daily at {scheduled_time} UTC.") + schedule.every().day.at(scheduled_time).do(lambda: asyncio.create_task(job())) + next_run_time = schedule.next_run() + logger.info(f"The next job is scheduled to run at {next_run_time}") + while True: await schedule.run_pending() await asyncio.sleep(1) From cd7833a82150f71a682e1a57ca6157c148df11b3 Mon Sep 17 00:00:00 2001 From: berkingurcan Date: Sun, 6 Oct 2024 13:16:00 +0700 Subject: [PATCH 10/10] black --- github_tracker_bot/bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github_tracker_bot/bot.py b/github_tracker_bot/bot.py index c66a7aa..2016850 100644 --- a/github_tracker_bot/bot.py +++ b/github_tracker_bot/bot.py @@ -108,7 +108,7 @@ async def job(): schedule.every().day.at(scheduled_time).do(lambda: asyncio.create_task(job())) next_run_time = schedule.next_run() logger.info(f"The next job is scheduled to run at {next_run_time}") - + while True: await schedule.run_pending() await asyncio.sleep(1)