fix(astrolabe): add locking to prevent token refresh race condition
Adds distributed locking using Nextcloud's ILockingProvider to prevent race conditions between background job and on-demand token refresh. Uses double-check locking pattern: 1. Quick check without lock - return immediately if token is valid 2. Acquire exclusive lock if token needs refresh 3. Re-check after lock - another process may have refreshed 4. Refresh only if still needed 5. Graceful degradation on LockedException Changes: - McpTokenStorage: add ILockingProvider, withTokenLock() method - McpTokenStorage: update getAccessToken() with locking pattern - RefreshUserTokens: wrap refresh in withTokenLock(), catch LockedException - Add comprehensive unit tests for locking behavior Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Chris Coutinho
co-authored by
Claude Opus 4.5
Claude Opus 4.5
parent
815a09be34
commit
5b71ac3251
+73
-32
@@ -9,6 +9,7 @@ use OCA\Astrolabe\Service\McpTokenStorage;
|
||||
use OCP\AppFramework\Utility\ITimeFactory;
|
||||
use OCP\BackgroundJob\IJob;
|
||||
use OCP\BackgroundJob\TimedJob;
|
||||
use OCP\Lock\LockedException;
|
||||
use Psr\Log\LoggerInterface;
|
||||
|
||||
/**
|
||||
@@ -76,6 +77,10 @@ class RefreshUserTokens extends TimedJob {
|
||||
* Calculates the refresh threshold based on the token's actual lifetime,
|
||||
* refreshing when less than 50% of the lifetime remains.
|
||||
*
|
||||
* Uses locking to prevent race conditions with on-demand refresh in
|
||||
* getAccessToken(). If lock cannot be acquired, skips this user since
|
||||
* on-demand refresh is already handling it.
|
||||
*
|
||||
* @return string 'refreshed', 'failed', or 'skipped'
|
||||
*/
|
||||
private function refreshUserTokenIfNeeded(string $userId): string {
|
||||
@@ -108,45 +113,81 @@ class RefreshUserTokens extends TimedJob {
|
||||
return 'skipped';
|
||||
}
|
||||
|
||||
// Token is expiring soon, attempt refresh
|
||||
if (!isset($token['refresh_token'])) {
|
||||
$this->logger->warning("RefreshUserTokens: User $userId has no refresh token");
|
||||
return 'failed';
|
||||
}
|
||||
|
||||
$this->logger->debug("RefreshUserTokens: Refreshing token for user $userId (remaining={$timeRemaining}s, threshold={$threshold}s)");
|
||||
|
||||
// Token is expiring soon, attempt refresh with lock
|
||||
try {
|
||||
/** @var string $refreshToken */
|
||||
$refreshToken = $token['refresh_token'];
|
||||
$newTokenData = $this->tokenRefresher->refreshAccessToken($refreshToken);
|
||||
return $this->tokenStorage->withTokenLock($userId, function () use ($userId) {
|
||||
// Re-check token after acquiring lock (double-check pattern)
|
||||
// Another process may have refreshed while we waited for lock
|
||||
$currentToken = $this->tokenStorage->getUserToken($userId);
|
||||
|
||||
if ($newTokenData === null) {
|
||||
$this->logger->warning("RefreshUserTokens: Refresh returned null for user $userId");
|
||||
// Don't delete token here - let on-demand refresh handle cleanup
|
||||
return 'failed';
|
||||
}
|
||||
if ($currentToken === null) {
|
||||
return 'skipped';
|
||||
}
|
||||
|
||||
// Calculate new expiration and store issued_at for future calculations
|
||||
$expiresIn = (int)($newTokenData['expires_in'] ?? self::DEFAULT_TOKEN_LIFETIME_SECONDS);
|
||||
$now = time();
|
||||
// Recalculate threshold with current token data
|
||||
$currentExpiresAt = (int)($currentToken['expires_at'] ?? 0);
|
||||
$currentIssuedAt = isset($currentToken['issued_at']) ? (int)$currentToken['issued_at'] : null;
|
||||
$currentTimeRemaining = $currentExpiresAt - time();
|
||||
|
||||
/** @var string $accessToken */
|
||||
$accessToken = $newTokenData['access_token'];
|
||||
/** @var string $newRefreshToken */
|
||||
$newRefreshToken = $newTokenData['refresh_token'] ?? $refreshToken;
|
||||
if ($currentIssuedAt !== null) {
|
||||
$currentTokenLifetime = $currentExpiresAt - $currentIssuedAt;
|
||||
} else {
|
||||
$currentTokenLifetime = self::DEFAULT_TOKEN_LIFETIME_SECONDS;
|
||||
}
|
||||
|
||||
$this->tokenStorage->storeUserToken(
|
||||
$userId,
|
||||
$accessToken,
|
||||
$newRefreshToken,
|
||||
$now + $expiresIn,
|
||||
$now // issued_at
|
||||
);
|
||||
$currentThreshold = max(
|
||||
(int)($currentTokenLifetime * self::REFRESH_AT_REMAINING_PERCENT),
|
||||
self::MIN_THRESHOLD_SECONDS
|
||||
);
|
||||
|
||||
$this->logger->debug("RefreshUserTokens: Successfully refreshed token for user $userId");
|
||||
return 'refreshed';
|
||||
if ($currentTimeRemaining > $currentThreshold) {
|
||||
// Token was refreshed by another process while we waited
|
||||
$this->logger->debug("RefreshUserTokens: Token already refreshed for user $userId while waiting for lock");
|
||||
return 'skipped';
|
||||
}
|
||||
|
||||
// Still needs refresh, proceed
|
||||
if (!isset($currentToken['refresh_token'])) {
|
||||
$this->logger->warning("RefreshUserTokens: User $userId has no refresh token");
|
||||
return 'failed';
|
||||
}
|
||||
|
||||
$this->logger->debug("RefreshUserTokens: Refreshing token for user $userId (remaining={$currentTimeRemaining}s, threshold={$currentThreshold}s)");
|
||||
|
||||
/** @var string $refreshToken */
|
||||
$refreshToken = $currentToken['refresh_token'];
|
||||
$newTokenData = $this->tokenRefresher->refreshAccessToken($refreshToken);
|
||||
|
||||
if ($newTokenData === null) {
|
||||
$this->logger->warning("RefreshUserTokens: Refresh returned null for user $userId");
|
||||
// Don't delete token here - let on-demand refresh handle cleanup
|
||||
return 'failed';
|
||||
}
|
||||
|
||||
// Calculate new expiration and store issued_at for future calculations
|
||||
$expiresIn = (int)($newTokenData['expires_in'] ?? self::DEFAULT_TOKEN_LIFETIME_SECONDS);
|
||||
$now = time();
|
||||
|
||||
/** @var string $accessToken */
|
||||
$accessToken = $newTokenData['access_token'];
|
||||
/** @var string $newRefreshToken */
|
||||
$newRefreshToken = $newTokenData['refresh_token'] ?? $refreshToken;
|
||||
|
||||
$this->tokenStorage->storeUserToken(
|
||||
$userId,
|
||||
$accessToken,
|
||||
$newRefreshToken,
|
||||
$now + $expiresIn,
|
||||
$now // issued_at
|
||||
);
|
||||
|
||||
$this->logger->debug("RefreshUserTokens: Successfully refreshed token for user $userId");
|
||||
return 'refreshed';
|
||||
});
|
||||
} catch (LockedException $e) {
|
||||
// Lock held by on-demand refresh - expected, not an error
|
||||
$this->logger->debug("RefreshUserTokens: Lock held for user $userId, skipping");
|
||||
return 'skipped';
|
||||
} catch (\Exception $e) {
|
||||
$this->logger->error("RefreshUserTokens: Failed to refresh for user $userId: " . $e->getMessage());
|
||||
return 'failed';
|
||||
|
||||
+109
-41
@@ -6,6 +6,8 @@ namespace OCA\Astrolabe\Service;
|
||||
|
||||
use OCP\IConfig;
|
||||
use OCP\IDBConnection;
|
||||
use OCP\Lock\ILockingProvider;
|
||||
use OCP\Lock\LockedException;
|
||||
use OCP\Security\ICrypto;
|
||||
use Psr\Log\LoggerInterface;
|
||||
|
||||
@@ -23,17 +25,20 @@ class McpTokenStorage {
|
||||
private $crypto;
|
||||
private $db;
|
||||
private $logger;
|
||||
private ILockingProvider $lockingProvider;
|
||||
|
||||
public function __construct(
|
||||
IConfig $config,
|
||||
ICrypto $crypto,
|
||||
IDBConnection $db,
|
||||
LoggerInterface $logger,
|
||||
ILockingProvider $lockingProvider,
|
||||
) {
|
||||
$this->config = $config;
|
||||
$this->crypto = $crypto;
|
||||
$this->db = $db;
|
||||
$this->logger = $logger;
|
||||
$this->lockingProvider = $lockingProvider;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -136,6 +141,38 @@ class McpTokenStorage {
|
||||
return time() >= ($token['expires_at'] - self::TOKEN_EXPIRY_BUFFER_SECONDS);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the lock path for a user's token refresh operation.
|
||||
*
|
||||
* @param string $userId User ID
|
||||
* @return string Lock path
|
||||
*/
|
||||
private function getTokenRefreshLockPath(string $userId): string {
|
||||
return 'astrolabe/oauth/tokens/' . $userId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute callback while holding exclusive lock on user's token.
|
||||
*
|
||||
* Prevents race conditions between background job and on-demand token refresh.
|
||||
*
|
||||
* @template T
|
||||
* @param string $userId User ID
|
||||
* @param callable(): T $callback
|
||||
* @return T
|
||||
* @throws LockedException If lock cannot be acquired
|
||||
*/
|
||||
public function withTokenLock(string $userId, callable $callback): mixed {
|
||||
$lockPath = $this->getTokenRefreshLockPath($userId);
|
||||
|
||||
$this->lockingProvider->acquireLock($lockPath, ILockingProvider::LOCK_EXCLUSIVE);
|
||||
try {
|
||||
return $callback();
|
||||
} finally {
|
||||
$this->lockingProvider->releaseLock($lockPath, ILockingProvider::LOCK_EXCLUSIVE);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete stored tokens for a user.
|
||||
*
|
||||
@@ -195,67 +232,98 @@ class McpTokenStorage {
|
||||
* This is a convenience method that combines token retrieval,
|
||||
* expiration checking, and automatic refresh if needed.
|
||||
*
|
||||
* Uses double-check locking pattern to prevent race conditions between
|
||||
* background job and on-demand refresh while minimizing lock contention.
|
||||
*
|
||||
* @param string $userId User ID
|
||||
* @param callable|null $refreshCallback Callback to refresh token if expired
|
||||
* Should accept (refreshToken) and return new token data
|
||||
* @return string|null Access token, or null if not available
|
||||
*/
|
||||
public function getAccessToken(string $userId, ?callable $refreshCallback = null): ?string {
|
||||
// Quick check without lock (optimization)
|
||||
$token = $this->getUserToken($userId);
|
||||
|
||||
if (!$token) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Check if token is expired
|
||||
if ($this->isExpired($token)) {
|
||||
// Try to refresh if callback provided
|
||||
if ($refreshCallback && isset($token['refresh_token'])) {
|
||||
try {
|
||||
$newTokenData = $refreshCallback($token['refresh_token']);
|
||||
// If not expired, return immediately without lock
|
||||
if (!$this->isExpired($token)) {
|
||||
return $token['access_token'];
|
||||
}
|
||||
|
||||
if ($newTokenData && isset($newTokenData['access_token'])) {
|
||||
// Store refreshed token
|
||||
// Use new refresh token if provided (rotation), otherwise keep old one
|
||||
$now = time();
|
||||
/** @var string $accessToken */
|
||||
$accessToken = $newTokenData['access_token'];
|
||||
/** @var string $refreshToken */
|
||||
$refreshToken = $newTokenData['refresh_token'] ?? $token['refresh_token'];
|
||||
$expiresIn = (int)($newTokenData['expires_in'] ?? 3600);
|
||||
// Token expired - acquire lock for refresh
|
||||
try {
|
||||
/** @var string|null */
|
||||
return $this->withTokenLock($userId, function () use ($userId, $refreshCallback): ?string {
|
||||
// Re-check after acquiring lock (double-check pattern)
|
||||
// Another process may have refreshed while we waited for the lock
|
||||
$currentToken = $this->getUserToken($userId);
|
||||
|
||||
$this->storeUserToken(
|
||||
$userId,
|
||||
$accessToken,
|
||||
$refreshToken,
|
||||
$now + $expiresIn,
|
||||
$now // issued_at for accurate lifetime calculation
|
||||
);
|
||||
|
||||
return $accessToken;
|
||||
}
|
||||
} catch (\Exception $e) {
|
||||
$this->logger->error("Failed to refresh token for user $userId", [
|
||||
'error' => $e->getMessage()
|
||||
]);
|
||||
// Delete stale token to prevent repeated refresh attempts
|
||||
$this->deleteUserToken($userId);
|
||||
if (!$currentToken) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Refresh callback returned null or invalid data - delete stale token
|
||||
// Check if another process already refreshed the token
|
||||
if (!$this->isExpired($currentToken)) {
|
||||
$this->logger->debug("Token already refreshed for user $userId while waiting for lock");
|
||||
return $currentToken['access_token'];
|
||||
}
|
||||
|
||||
// Still expired, perform refresh
|
||||
if ($refreshCallback && isset($currentToken['refresh_token'])) {
|
||||
try {
|
||||
/** @var string $refreshToken */
|
||||
$refreshToken = $currentToken['refresh_token'];
|
||||
$newTokenData = $refreshCallback($refreshToken);
|
||||
|
||||
if ($newTokenData && isset($newTokenData['access_token'])) {
|
||||
// Store refreshed token
|
||||
// Use new refresh token if provided (rotation), otherwise keep old one
|
||||
$now = time();
|
||||
/** @var string $accessToken */
|
||||
$accessToken = $newTokenData['access_token'];
|
||||
/** @var string $newRefreshToken */
|
||||
$newRefreshToken = $newTokenData['refresh_token'] ?? $refreshToken;
|
||||
$expiresIn = (int)($newTokenData['expires_in'] ?? 3600);
|
||||
|
||||
$this->storeUserToken(
|
||||
$userId,
|
||||
$accessToken,
|
||||
$newRefreshToken,
|
||||
$now + $expiresIn,
|
||||
$now // issued_at for accurate lifetime calculation
|
||||
);
|
||||
|
||||
return $accessToken;
|
||||
}
|
||||
} catch (\Exception $e) {
|
||||
$this->logger->error("Failed to refresh token for user $userId", [
|
||||
'error' => $e->getMessage()
|
||||
]);
|
||||
// Delete stale token to prevent repeated refresh attempts
|
||||
$this->deleteUserToken($userId);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Refresh callback returned null or invalid data - delete stale token
|
||||
$this->deleteUserToken($userId);
|
||||
$this->logger->info("Deleted stale token for user $userId after refresh failure");
|
||||
return null;
|
||||
}
|
||||
|
||||
// Token expired and no refresh callback available - delete stale token
|
||||
$this->deleteUserToken($userId);
|
||||
$this->logger->info("Deleted stale token for user $userId after refresh failure");
|
||||
$this->logger->info("Token expired for user $userId, no refresh available");
|
||||
return null;
|
||||
}
|
||||
|
||||
// Token expired and no refresh callback available - delete stale token
|
||||
$this->deleteUserToken($userId);
|
||||
$this->logger->info("Token expired for user $userId, no refresh available");
|
||||
return null;
|
||||
});
|
||||
} catch (LockedException $e) {
|
||||
// Could not acquire lock - another process is refreshing
|
||||
// Return stale token rather than failing - caller can retry if needed
|
||||
$this->logger->warning("Could not acquire token lock for user $userId, returning stale token");
|
||||
return $token['access_token'] ?? null;
|
||||
}
|
||||
|
||||
return $token['access_token'];
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user