diff --git a/internal/leetcode-auth-stealer-service/src/grpcserve/service/auth.py b/internal/leetcode-auth-stealer-service/src/grpcserve/service/auth.py index 3ff3a590f..4e354d56b 100644 --- a/internal/leetcode-auth-stealer-service/src/grpcserve/service/auth.py +++ b/internal/leetcode-auth-stealer-service/src/grpcserve/service/auth.py @@ -1,4 +1,5 @@ import asyncio +from collections.abc import Callable import grpc import structlog @@ -8,7 +9,7 @@ LeetcodeAuthStealerService_pb2, LeetcodeAuthStealerService_pb2_grpc, ) -from prom import record_auth_failure, record_auth_success +from prom import AuthFailureReason, record_auth_failure, record_auth_success from provider.camou import CamouProvider from provider.dto import ( AuthenticationTimeout, @@ -24,9 +25,23 @@ class AuthService( LeetcodeAuthStealerService_pb2_grpc.LeetcodeAuthStealerServiceServicer ): - def __init__(self, c: Config, provider: CamouProvider) -> None: + def __init__( + self, + c: Config, + provider: CamouProvider, + on_failure: Callable[[], None], + ) -> None: self.provider = provider self.timeout_seconds = c.auth_timeout_seconds + self.on_failure = on_failure + + def _fail(self, reason: AuthFailureReason) -> None: + record_auth_failure(reason) + + logger.warning( + "Authentication failed, marking service as unalive", reason=reason + ) + self.on_failure() async def Authenticate( self, @@ -45,36 +60,36 @@ async def Authenticate( Credentials(request.login, request.password) ) except BadCredentials: - record_auth_failure("bad_credentials") + self._fail("bad_credentials") await context.abort( grpc.StatusCode.UNAUTHENTICATED, "Invalid LeetCode credentials" ) except ManualVerificationRequired: - record_auth_failure("manual_verification_required") + self._fail("manual_verification_required") await context.abort( grpc.StatusCode.FAILED_PRECONDITION, "Manual two-step verification is required", ) except AuthenticationUnavailable: - record_auth_failure("authentication_unavailable") + self._fail("authentication_unavailable") await context.abort( grpc.StatusCode.UNAVAILABLE, "LeetCode authentication is unavailable" ) except AuthenticationTimeout, TimeoutError: - record_auth_failure("timeout") + self._fail("timeout") await context.abort( grpc.StatusCode.DEADLINE_EXCEEDED, "LeetCode authentication timed out" ) except asyncio.CancelledError: - record_auth_failure("cancelled") + self._fail("cancelled") raise except Exception: - record_auth_failure("internal_error") + self._fail("internal_error") logger.exception("Authenticate failed") await context.abort(grpc.StatusCode.INTERNAL, "Authenticate failed") if not cookies.leetcode_session or not cookies.csrf_token: - record_auth_failure("incomplete_authentication") + self._fail("incomplete_authentication") await context.abort( grpc.StatusCode.INTERNAL, "Provider returned incomplete authentication" ) diff --git a/internal/leetcode-auth-stealer-service/src/httpserve/health.py b/internal/leetcode-auth-stealer-service/src/httpserve/health.py index fbf63e067..881e7c48d 100644 --- a/internal/leetcode-auth-stealer-service/src/httpserve/health.py +++ b/internal/leetcode-auth-stealer-service/src/httpserve/health.py @@ -8,8 +8,13 @@ from httpserve.router import AppState -async def _livez(_: web.Request) -> web.Response: - return web.Response(text="OK", status=200) +async def _livez(request: web.Request) -> web.Response: + state: AppState = request.app["app_state"] + + return web.Response( + text="OK" if state.live else "mr stark i dont feel so good", + status=200 if state.live else 503, + ) async def _readyz(request: web.Request) -> web.Response: diff --git a/internal/leetcode-auth-stealer-service/src/httpserve/router.py b/internal/leetcode-auth-stealer-service/src/httpserve/router.py index 4f0b1ddcf..b8d4fb221 100644 --- a/internal/leetcode-auth-stealer-service/src/httpserve/router.py +++ b/internal/leetcode-auth-stealer-service/src/httpserve/router.py @@ -12,6 +12,14 @@ class AppState: cfg: Config ready: bool + # controlls /livez + # app may set this to False to trigger an unrecoverable state + # look at httpserve.health for how liveness works in k8s + live: bool = True + + # use to mark unalive without having to import/pass AppState + def mark_unalive(self) -> None: + self.live = False def register_routes(app: web.Application, state: AppState) -> web.Application: diff --git a/internal/leetcode-auth-stealer-service/src/server.py b/internal/leetcode-auth-stealer-service/src/server.py index 79b99d2e2..c9026e747 100644 --- a/internal/leetcode-auth-stealer-service/src/server.py +++ b/internal/leetcode-auth-stealer-service/src/server.py @@ -33,12 +33,13 @@ async def serve(c: Config) -> None: server = grpc.aio.server( interceptors=[RequestIdInterceptor()], # more than 1 will crash the pod due to multiple browsers - maximum_concurrent_rpcs=1 + maximum_concurrent_rpcs=1, ) invis_provider = CamouProvider(c) + app_state = AppState(c, ready=False) LeetcodeAuthStealerService_pb2_grpc.add_LeetcodeAuthStealerServiceServicer_to_server( - AuthService(c, invis_provider), server + AuthService(c, invis_provider, app_state.mark_unalive), server ) if c.grpc_reflection: @@ -53,7 +54,6 @@ async def serve(c: Config) -> None: if server.add_insecure_port(address) == 0: raise RuntimeError(f"Unable to bind gRPC server to {address}") - app_state = AppState(c, ready=False) app = register_routes(web.Application(), app_state) runner = web.AppRunner(app) diff --git a/src/main/java/org/patinanetwork/codebloom/scheduled/auth/LeetcodeAuthStealer.java b/src/main/java/org/patinanetwork/codebloom/scheduled/auth/LeetcodeAuthStealer.java index c0f2006a1..2baf14c47 100644 --- a/src/main/java/org/patinanetwork/codebloom/scheduled/auth/LeetcodeAuthStealer.java +++ b/src/main/java/org/patinanetwork/codebloom/scheduled/auth/LeetcodeAuthStealer.java @@ -62,7 +62,7 @@ public LeetcodeAuthStealer( * authentication key from Leetcode. That code is stored in the database and can then be used to run authenticated * queries such as being used to retrieve code from our user submissions. */ - @Scheduled(initialDelay = 0, fixedDelay = 1, timeUnit = TimeUnit.HOURS) + @Scheduled(initialDelay = 0, fixedDelay = 15, timeUnit = TimeUnit.MINUTES) public void stealAuthCookie() { boolean acquired = LOCK.writeLock().tryLock(); if (!acquired) {