From 8b6ebbe81511c8331111b97b1281b5bcdcff8594 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 14 Mar 2023 16:27:40 +0100 Subject: [PATCH] feat(chalice): upgraded startup/shutdown logic feat(alerts): health-check endpoint --- api/app_alerts.py | 64 ++++++++++++++------------- api/chalicelib/core/health.py | 2 +- api/entrypoint.sh | 2 +- api/entrypoint_alerts.sh | 2 +- api/run-alerts-dev.sh | 2 +- ee/api/.gitignore | 2 + ee/api/app.py | 81 +++++++++++++++++------------------ ee/api/clean-dev.sh | 2 + ee/api/entrypoint.sh | 2 +- ee/api/entrypoint_alerts.sh | 2 +- ee/api/run-dev.sh | 3 -- 11 files changed, 83 insertions(+), 81 deletions(-) delete mode 100755 ee/api/run-dev.sh diff --git a/api/app_alerts.py b/api/app_alerts.py index 111bad2a1..02147ef23 100644 --- a/api/app_alerts.py +++ b/api/app_alerts.py @@ -1,33 +1,17 @@ import logging +from contextlib import asynccontextmanager from apscheduler.schedulers.asyncio import AsyncIOScheduler from decouple import config from fastapi import FastAPI -from chalicelib.utils import pg_client from chalicelib.core import alerts_processor - -app = FastAPI(root_path="/alerts", docs_url=config("docs_url", default=""), redoc_url=config("redoc_url", default="")) -logging.info("============= ALERTS =============") +from chalicelib.utils import pg_client -@app.get("/") -async def root(): - return {"status": "Running"} - - -app.schedule = AsyncIOScheduler() - -loglevel = config("LOGLEVEL", default=logging.INFO) -print(f">Loglevel set to: {loglevel}") -logging.basicConfig(level=loglevel) -ap_logger = logging.getLogger('apscheduler') -ap_logger.setLevel(loglevel) -app.schedule = AsyncIOScheduler() - - -@app.on_event("startup") -async def startup(): +@asynccontextmanager +async def lifespan(app: FastAPI): + # Startup logging.info(">>>>> starting up <<<<<") await pg_client.init() app.schedule.start() @@ -39,24 +23,44 @@ async def startup(): for job in app.schedule.get_jobs(): ap_logger.info({"Name": str(job.id), "Run Frequency": str(job.trigger), "Next Run": str(job.next_run_time)}) + # App listening + yield -@app.on_event("shutdown") -async def shutdown(): + # Shutdown logging.info(">>>>> shutting down <<<<<") app.schedule.shutdown(wait=False) await pg_client.terminate() -@app.get('/private/shutdown', tags=["private"]) -async def stop_server(): - logging.info("Requested shutdown") - await shutdown() - import os, signal - os.kill(1, signal.SIGTERM) +app = FastAPI(root_path="/alerts", docs_url=config("docs_url", default=""), redoc_url=config("redoc_url", default=""), + lifespan=lifespan) +logging.info("============= ALERTS =============") +@app.get("/") +async def root(): + return {"status": "Running"} + + +@app.get("/health") +async def get_health_status(): + return {"data": { + "health": True, + "details": {"version": config("version_number", default="unknown")} + }} + + +app.schedule = AsyncIOScheduler() + +loglevel = config("LOGLEVEL", default=logging.INFO) +print(f">Loglevel set to: {loglevel}") +logging.basicConfig(level=loglevel) +ap_logger = logging.getLogger('apscheduler') +ap_logger.setLevel(loglevel) +app.schedule = AsyncIOScheduler() + if config("LOCAL_DEV", default=False, cast=bool): - @app.get('/private/trigger', tags=["private"]) + @app.get('/trigger', tags=["private"]) async def trigger_main_cron(): logging.info("Triggering main cron") alerts_processor.process() diff --git a/api/chalicelib/core/health.py b/api/chalicelib/core/health.py index 980c7a41b..6f236a0f5 100644 --- a/api/chalicelib/core/health.py +++ b/api/chalicelib/core/health.py @@ -29,7 +29,7 @@ if config("LOCAL_DEV", cast=bool, default=False): else: HEALTH_ENDPOINTS = { - "alerts": "http://alerts-openreplay.app.svc.cluster.local:8888/metrics", + "alerts": "http://alerts-openreplay.app.svc.cluster.local:8888/health", "assets": "http://assets-openreplay.app.svc.cluster.local:8888/metrics", "assist": "http://assist-openreplay.app.svc.cluster.local:8888/health", "chalice": "http://chalice-openreplay.app.svc.cluster.local:8888/metrics", diff --git a/api/entrypoint.sh b/api/entrypoint.sh index e140268ef..401046526 100755 --- a/api/entrypoint.sh +++ b/api/entrypoint.sh @@ -1,3 +1,3 @@ #!/bin/sh -uvicorn app:app --host 0.0.0.0 --port $LISTEN_PORT --reload --proxy-headers +uvicorn app:app --host 0.0.0.0 --port $LISTEN_PORT --proxy-headers diff --git a/api/entrypoint_alerts.sh b/api/entrypoint_alerts.sh index dedfa102b..9ac93dd60 100755 --- a/api/entrypoint_alerts.sh +++ b/api/entrypoint_alerts.sh @@ -1,3 +1,3 @@ #!/bin/sh export ASSIST_KEY=ignore -uvicorn app:app --host 0.0.0.0 --port $LISTEN_PORT --reload +uvicorn app:app --host 0.0.0.0 --port 8888 diff --git a/api/run-alerts-dev.sh b/api/run-alerts-dev.sh index 54db30171..309356133 100755 --- a/api/run-alerts-dev.sh +++ b/api/run-alerts-dev.sh @@ -1,3 +1,3 @@ #!/bin/zsh -uvicorn app_alerts:app --reload \ No newline at end of file +uvicorn app_alerts:app --reload --port 8888 \ No newline at end of file diff --git a/ee/api/.gitignore b/ee/api/.gitignore index 27ac41f5c..1e342e8bc 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -265,6 +265,8 @@ Pipfile.lock /app_alerts.py /build_alerts.sh /build_crons.sh +/run-dev.sh +/run-alerts-dev.sh /routers/subs/health.py /routers/subs/v1_api.py #exp /chalicelib/core/dashboards.py diff --git a/ee/api/app.py b/ee/api/app.py index 407e4aa5b..9104a2db4 100644 --- a/ee/api/app.py +++ b/ee/api/app.py @@ -1,5 +1,6 @@ import logging import queue +from contextlib import asynccontextmanager from apscheduler.schedulers.asyncio import AsyncIOScheduler from decouple import config @@ -10,9 +11,9 @@ from starlette import status from starlette.responses import StreamingResponse, JSONResponse from chalicelib.core import traces +from chalicelib.utils import events_queue from chalicelib.utils import helper from chalicelib.utils import pg_client -from chalicelib.utils import events_queue from routers import core, core_dynamic, ee, saml from routers.crons import core_crons from routers.crons import core_dynamic_crons @@ -20,7 +21,43 @@ from routers.crons import ee_crons from routers.subs import insights, metrics, v1_api_ee from routers.subs import v1_api, health -app = FastAPI(root_path="/api", docs_url=config("docs_url", default=""), redoc_url=config("redoc_url", default="")) +loglevel = config("LOGLEVEL", default=logging.INFO) +print(f">Loglevel set to: {loglevel}") +logging.basicConfig(level=loglevel) +ap_logger = logging.getLogger('apscheduler') +ap_logger.setLevel(loglevel) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + # Startup + logging.info(">>>>> starting up <<<<<") + app.schedule = AsyncIOScheduler() + app.queue_system = queue.Queue() + await pg_client.init() + await events_queue.init() + app.schedule.start() + + for job in core_crons.cron_jobs + core_dynamic_crons.cron_jobs + traces.cron_jobs + ee_crons.ee_cron_jobs: + app.schedule.add_job(id=job["func"].__name__, **job) + + ap_logger.info(">Scheduled jobs:") + for job in app.schedule.get_jobs(): + ap_logger.info({"Name": str(job.id), "Run Frequency": str(job.trigger), "Next Run": str(job.next_run_time)}) + + # App listening + yield + + # Shutdown + logging.info(">>>>> shutting down <<<<<") + app.schedule.shutdown(wait=True) + await traces.process_traces_queue() + await events_queue.terminate() + await pg_client.terminate() + + +app = FastAPI(root_path="/api", docs_url=config("docs_url", default=""), redoc_url=config("redoc_url", default=""), + lifespan=lifespan) app.add_middleware(GZipMiddleware, minimum_size=1000) @@ -71,43 +108,3 @@ app.include_router(v1_api_ee.app_apikey) app.include_router(health.public_app) app.include_router(health.app) app.include_router(health.app_apikey) - -loglevel = config("LOGLEVEL", default=logging.INFO) -print(f">Loglevel set to: {loglevel}") -logging.basicConfig(level=loglevel) -ap_logger = logging.getLogger('apscheduler') -ap_logger.setLevel(loglevel) -app.schedule = AsyncIOScheduler() -app.queue_system = queue.Queue() - - -@app.on_event("startup") -async def startup(): - logging.info(">>>>> starting up <<<<<") - await pg_client.init() - await events_queue.init() - app.schedule.start() - - for job in core_crons.cron_jobs + core_dynamic_crons.cron_jobs + traces.cron_jobs + ee_crons.ee_cron_jobs: - app.schedule.add_job(id=job["func"].__name__, **job) - - ap_logger.info(">Scheduled jobs:") - for job in app.schedule.get_jobs(): - ap_logger.info({"Name": str(job.id), "Run Frequency": str(job.trigger), "Next Run": str(job.next_run_time)}) - - -@app.on_event("shutdown") -async def shutdown(): - logging.info(">>>>> shutting down <<<<<") - app.schedule.shutdown(wait=True) - await traces.process_traces_queue() - await events_queue.terminate() - await pg_client.terminate() - - -@app.get('/private/shutdown', tags=["private"]) -async def stop_server(): - logging.info("Requested shutdown") - await shutdown() - import os, signal - os.kill(1, signal.SIGTERM) diff --git a/ee/api/clean-dev.sh b/ee/api/clean-dev.sh index a160cf9c2..c47a80ee8 100755 --- a/ee/api/clean-dev.sh +++ b/ee/api/clean-dev.sh @@ -86,3 +86,5 @@ rm -rf ./chalicelib/core/performance_event.py rm -rf ./chalicelib/core/saved_search.py rm -rf ./app_alerts.py rm -rf ./build_alerts.sh +rm -rf ./run-dev.sh +rm -rf ./run-alerts-dev.sh diff --git a/ee/api/entrypoint.sh b/ee/api/entrypoint.sh index ebd646a7d..e63d4e2af 100755 --- a/ee/api/entrypoint.sh +++ b/ee/api/entrypoint.sh @@ -2,4 +2,4 @@ sh env_vars.sh source /tmp/.env.override -uvicorn app:app --host 0.0.0.0 --port $LISTEN_PORT --reload --proxy-headers +uvicorn app:app --host 0.0.0.0 --port $LISTEN_PORT --proxy-headers diff --git a/ee/api/entrypoint_alerts.sh b/ee/api/entrypoint_alerts.sh index acf8b390a..410015142 100755 --- a/ee/api/entrypoint_alerts.sh +++ b/ee/api/entrypoint_alerts.sh @@ -2,4 +2,4 @@ export ASSIST_KEY=ignore sh env_vars.sh source /tmp/.env.override -uvicorn app:app --host 0.0.0.0 --port $LISTEN_PORT --reload +uvicorn app:app --host 0.0.0.0 --port 8888 diff --git a/ee/api/run-dev.sh b/ee/api/run-dev.sh deleted file mode 100755 index 76682286d..000000000 --- a/ee/api/run-dev.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/zsh - -uvicorn app:app --reload \ No newline at end of file