Implement non-redis lock

This commit is contained in:
Owen
2026-06-24 16:00:54 -04:00
parent 2b38658ea6
commit 242123b875
2 changed files with 222 additions and 55 deletions
+117 -7
View File
@@ -1,4 +1,24 @@
const instanceId = `local-${Math.random().toString(36).slice(2)}-${Date.now()}`;
type LocalLockRecord = {
owner: string;
expiresAt: number;
};
const localLocks = new Map<string, LocalLockRecord>();
export class LockManager { export class LockManager {
private clearExpiredLocalLock(lockKey: string): void {
const current = localLocks.get(lockKey);
if (current && current.expiresAt <= Date.now()) {
localLocks.delete(lockKey);
}
}
private getLocalOwnerToken(): string {
return `${instanceId}:`;
}
/** /**
* Acquire a distributed lock using Redis SET with NX and PX options * Acquire a distributed lock using Redis SET with NX and PX options
* @param lockKey - Unique identifier for the lock * @param lockKey - Unique identifier for the lock
@@ -7,22 +27,57 @@ export class LockManager {
*/ */
async acquireLock( async acquireLock(
lockKey: string, lockKey: string,
ttlMs: number = 30000 ttlMs: number = 30000,
maxRetries: number = 3,
retryDelayMs: number = 100
): Promise<boolean> { ): Promise<boolean> {
return true; for (let attempt = 0; attempt < maxRetries; attempt++) {
this.clearExpiredLocalLock(lockKey);
const existing = localLocks.get(lockKey);
if (!existing) {
localLocks.set(lockKey, {
owner: this.getLocalOwnerToken(),
expiresAt: Date.now() + ttlMs
});
return true;
}
if (existing.owner === this.getLocalOwnerToken()) {
existing.expiresAt = Date.now() + ttlMs;
localLocks.set(lockKey, existing);
return true;
}
if (attempt < maxRetries - 1) {
const delay = retryDelayMs * Math.pow(2, attempt);
await new Promise((resolve) => setTimeout(resolve, delay));
}
}
return false;
} }
/** /**
* Release a lock using Lua script to ensure atomicity * Release a lock using Lua script to ensure atomicity
* @param lockKey - Unique identifier for the lock * @param lockKey - Unique identifier for the lock
*/ */
async releaseLock(lockKey: string): Promise<void> {} async releaseLock(lockKey: string): Promise<void> {
this.clearExpiredLocalLock(lockKey);
const existing = localLocks.get(lockKey);
if (existing && existing.owner === this.getLocalOwnerToken()) {
localLocks.delete(lockKey);
}
}
/** /**
* Force release a lock regardless of owner (use with caution) * Force release a lock regardless of owner (use with caution)
* @param lockKey - Unique identifier for the lock * @param lockKey - Unique identifier for the lock
*/ */
async forceReleaseLock(lockKey: string): Promise<void> {} async forceReleaseLock(lockKey: string): Promise<void> {
localLocks.delete(lockKey);
}
/** /**
* Check if a lock exists and get its info * Check if a lock exists and get its info
@@ -35,7 +90,20 @@ export class LockManager {
ttl: number; ttl: number;
owner?: string; owner?: string;
}> { }> {
return { exists: false, ownedByMe: false, ttl: 0 }; this.clearExpiredLocalLock(lockKey);
const existing = localLocks.get(lockKey);
if (!existing) {
return { exists: false, ownedByMe: false, ttl: 0 };
}
const ttl = Math.max(0, existing.expiresAt - Date.now());
return {
exists: true,
ownedByMe: existing.owner === this.getLocalOwnerToken(),
ttl,
owner: existing.owner.split(":")[0]
};
} }
/** /**
@@ -45,6 +113,15 @@ export class LockManager {
* @returns Promise<boolean> - true if extended successfully * @returns Promise<boolean> - true if extended successfully
*/ */
async extendLock(lockKey: string, ttlMs: number): Promise<boolean> { async extendLock(lockKey: string, ttlMs: number): Promise<boolean> {
this.clearExpiredLocalLock(lockKey);
const existing = localLocks.get(lockKey);
if (!existing || existing.owner !== this.getLocalOwnerToken()) {
return false;
}
existing.expiresAt = Date.now() + ttlMs;
localLocks.set(lockKey, existing);
return true; return true;
} }
@@ -62,7 +139,26 @@ export class LockManager {
maxRetries: number = 5, maxRetries: number = 5,
baseDelayMs: number = 100 baseDelayMs: number = 100
): Promise<boolean> { ): Promise<boolean> {
return true; for (let attempt = 0; attempt <= maxRetries; attempt++) {
const acquired = await this.acquireLock(
lockKey,
ttlMs,
1,
baseDelayMs
);
if (acquired) {
return true;
}
if (attempt < maxRetries) {
const delay =
baseDelayMs * Math.pow(2, attempt) + Math.random() * 100;
await new Promise((resolve) => setTimeout(resolve, delay));
}
}
return false;
} }
/** /**
@@ -99,7 +195,21 @@ export class LockManager {
activeLocksCount: number; activeLocksCount: number;
locksOwnedByMe: number; locksOwnedByMe: number;
}> { }> {
return { activeLocksCount: 0, locksOwnedByMe: 0 }; const now = Date.now();
for (const [key, value] of localLocks.entries()) {
if (value.expiresAt <= now) {
localLocks.delete(key);
}
}
let locksOwnedByMe = 0;
for (const value of localLocks.values()) {
if (value.owner === this.getLocalOwnerToken()) {
locksOwnedByMe++;
}
}
return { activeLocksCount: localLocks.size, locksOwnedByMe };
} }
/** /**
+105 -48
View File
@@ -11,14 +11,31 @@
* This file is not licensed under the AGPLv3. * This file is not licensed under the AGPLv3.
*/ */
import { config } from "@server/lib/config";
import logger from "@server/logger"; import logger from "@server/logger";
import { redis } from "#private/lib/redis"; import { redis } from "#private/lib/redis";
import { v4 as uuidv4 } from "uuid"; import { v4 as uuidv4 } from "uuid";
const instanceId = uuidv4(); const instanceId = uuidv4();
type LocalLockRecord = {
owner: string;
expiresAt: number;
};
const localLocks = new Map<string, LocalLockRecord>();
export class LockManager { export class LockManager {
private clearExpiredLocalLock(lockKey: string): void {
const current = localLocks.get(lockKey);
if (current && current.expiresAt <= Date.now()) {
localLocks.delete(lockKey);
}
}
private getLocalOwnerToken(): string {
return `${instanceId}:`;
}
/** /**
* Acquire a distributed lock using Redis SET with NX and PX options * Acquire a distributed lock using Redis SET with NX and PX options
* @param lockKey - Unique identifier for the lock * @param lockKey - Unique identifier for the lock
@@ -32,12 +49,34 @@ export class LockManager {
retryDelayMs: number = 100 retryDelayMs: number = 100
): Promise<boolean> { ): Promise<boolean> {
if (!redis || !redis.status || redis.status !== "ready") { if (!redis || !redis.status || redis.status !== "ready") {
return true; for (let attempt = 0; attempt < maxRetries; attempt++) {
this.clearExpiredLocalLock(lockKey);
const existing = localLocks.get(lockKey);
if (!existing) {
localLocks.set(lockKey, {
owner: this.getLocalOwnerToken(),
expiresAt: Date.now() + ttlMs
});
return true;
}
if (existing.owner === this.getLocalOwnerToken()) {
existing.expiresAt = Date.now() + ttlMs;
localLocks.set(lockKey, existing);
return true;
}
if (attempt < maxRetries - 1) {
const delay = retryDelayMs * Math.pow(2, attempt);
await new Promise((resolve) => setTimeout(resolve, delay));
}
}
return false;
} }
const lockValue = `${ const lockValue = `${instanceId}:${Date.now()}`;
instanceId
}:${Date.now()}`;
const redisKey = `lock:${lockKey}`; const redisKey = `lock:${lockKey}`;
for (let attempt = 0; attempt < maxRetries; attempt++) { for (let attempt = 0; attempt < maxRetries; attempt++) {
@@ -53,11 +92,7 @@ export class LockManager {
); );
if (result === "OK") { if (result === "OK") {
logger.debug( logger.debug(`Lock acquired: ${lockKey} by ${instanceId}`);
`Lock acquired: ${lockKey} by ${
instanceId
}`
);
return true; return true;
} }
@@ -65,17 +100,11 @@ export class LockManager {
const existingValue = await redis.get(redisKey); const existingValue = await redis.get(redisKey);
if ( if (
existingValue && existingValue &&
existingValue.startsWith( existingValue.startsWith(`${instanceId}:`)
`${instanceId}:`
)
) { ) {
// Extend the lock TTL since it's the same worker // Extend the lock TTL since it's the same worker
await redis.pexpire(redisKey, ttlMs); await redis.pexpire(redisKey, ttlMs);
logger.debug( logger.debug(`Lock extended: ${lockKey} by ${instanceId}`);
`Lock extended: ${lockKey} by ${
instanceId
}`
);
return true; return true;
} }
@@ -88,7 +117,10 @@ export class LockManager {
await new Promise((resolve) => setTimeout(resolve, delay)); await new Promise((resolve) => setTimeout(resolve, delay));
} }
} catch (error) { } catch (error) {
logger.error(`Failed to acquire lock ${lockKey} (attempt ${attempt + 1}/${maxRetries}):`, error); logger.error(
`Failed to acquire lock ${lockKey} (attempt ${attempt + 1}/${maxRetries}):`,
error
);
// On error, still retry if we have attempts left // On error, still retry if we have attempts left
if (attempt < maxRetries - 1) { if (attempt < maxRetries - 1) {
const delay = retryDelayMs * Math.pow(2, attempt); const delay = retryDelayMs * Math.pow(2, attempt);
@@ -109,6 +141,11 @@ export class LockManager {
*/ */
async releaseLock(lockKey: string): Promise<void> { async releaseLock(lockKey: string): Promise<void> {
if (!redis || !redis.status || redis.status !== "ready") { if (!redis || !redis.status || redis.status !== "ready") {
this.clearExpiredLocalLock(lockKey);
const existing = localLocks.get(lockKey);
if (existing && existing.owner === this.getLocalOwnerToken()) {
localLocks.delete(lockKey);
}
return; return;
} }
@@ -136,11 +173,7 @@ export class LockManager {
)) as number; )) as number;
if (result === 1) { if (result === 1) {
logger.debug( logger.debug(`Lock released: ${lockKey} by ${instanceId}`);
`Lock released: ${lockKey} by ${
instanceId
}`
);
} else { } else {
logger.warn( logger.warn(
`Lock not released - not owned by worker: ${lockKey} by ${ `Lock not released - not owned by worker: ${lockKey} by ${
@@ -159,6 +192,7 @@ export class LockManager {
*/ */
async forceReleaseLock(lockKey: string): Promise<void> { async forceReleaseLock(lockKey: string): Promise<void> {
if (!redis || !redis.status || redis.status !== "ready") { if (!redis || !redis.status || redis.status !== "ready") {
localLocks.delete(lockKey);
return; return;
} }
@@ -186,7 +220,20 @@ export class LockManager {
owner?: string; owner?: string;
}> { }> {
if (!redis || !redis.status || redis.status !== "ready") { if (!redis || !redis.status || redis.status !== "ready") {
return { exists: false, ownedByMe: true, ttl: 0 }; this.clearExpiredLocalLock(lockKey);
const existing = localLocks.get(lockKey);
if (!existing) {
return { exists: false, ownedByMe: false, ttl: 0 };
}
const ttl = Math.max(0, existing.expiresAt - Date.now());
return {
exists: true,
ownedByMe: existing.owner === this.getLocalOwnerToken(),
ttl,
owner: existing.owner.split(":")[0]
};
} }
const redisKey = `lock:${lockKey}`; const redisKey = `lock:${lockKey}`;
@@ -198,11 +245,7 @@ export class LockManager {
]); ]);
const exists = value !== null; const exists = value !== null;
const ownedByMe = const ownedByMe = exists && value!.startsWith(`${instanceId}:`);
exists &&
value!.startsWith(
`${instanceId}:`
);
const owner = exists ? value!.split(":")[0] : undefined; const owner = exists ? value!.split(":")[0] : undefined;
return { return {
@@ -225,6 +268,15 @@ export class LockManager {
*/ */
async extendLock(lockKey: string, ttlMs: number): Promise<boolean> { async extendLock(lockKey: string, ttlMs: number): Promise<boolean> {
if (!redis || !redis.status || redis.status !== "ready") { if (!redis || !redis.status || redis.status !== "ready") {
this.clearExpiredLocalLock(lockKey);
const existing = localLocks.get(lockKey);
if (!existing || existing.owner !== this.getLocalOwnerToken()) {
return false;
}
existing.expiresAt = Date.now() + ttlMs;
localLocks.set(lockKey, existing);
return true; return true;
} }
@@ -255,9 +307,7 @@ export class LockManager {
if (result === 1) { if (result === 1) {
logger.debug( logger.debug(
`Lock extended: ${lockKey} by ${ `Lock extended: ${lockKey} by ${instanceId} for ${ttlMs}ms`
instanceId
} for ${ttlMs}ms`
); );
return true; return true;
} }
@@ -282,12 +332,13 @@ export class LockManager {
maxRetries: number = 5, maxRetries: number = 5,
baseDelayMs: number = 100 baseDelayMs: number = 100
): Promise<boolean> { ): Promise<boolean> {
if (!redis || !redis.status || redis.status !== "ready") {
return true;
}
for (let attempt = 0; attempt <= maxRetries; attempt++) { for (let attempt = 0; attempt <= maxRetries; attempt++) {
const acquired = await this.acquireLock(lockKey, ttlMs); const acquired = await this.acquireLock(
lockKey,
ttlMs,
1,
baseDelayMs
);
if (acquired) { if (acquired) {
return true; return true;
@@ -319,10 +370,6 @@ export class LockManager {
fn: () => Promise<T>, fn: () => Promise<T>,
ttlMs: number = 30000 ttlMs: number = 30000
): Promise<T> { ): Promise<T> {
if (!redis || !redis.status || redis.status !== "ready") {
return await fn();
}
const acquired = await this.acquireLock(lockKey, ttlMs); const acquired = await this.acquireLock(lockKey, ttlMs);
if (!acquired) { if (!acquired) {
@@ -346,7 +393,21 @@ export class LockManager {
locksOwnedByMe: number; locksOwnedByMe: number;
}> { }> {
if (!redis || !redis.status || redis.status !== "ready") { if (!redis || !redis.status || redis.status !== "ready") {
return { activeLocksCount: 0, locksOwnedByMe: 0 }; const now = Date.now();
for (const [key, value] of localLocks.entries()) {
if (value.expiresAt <= now) {
localLocks.delete(key);
}
}
let locksOwnedByMe = 0;
for (const value of localLocks.values()) {
if (value.owner === this.getLocalOwnerToken()) {
locksOwnedByMe++;
}
}
return { activeLocksCount: localLocks.size, locksOwnedByMe };
} }
try { try {
@@ -356,11 +417,7 @@ export class LockManager {
if (keys.length > 0) { if (keys.length > 0) {
const values = await redis.mget(...keys); const values = await redis.mget(...keys);
locksOwnedByMe = values.filter( locksOwnedByMe = values.filter(
(value) => (value) => value && value.startsWith(`${instanceId}:`)
value &&
value.startsWith(
`${instanceId}:`
)
).length; ).length;
} }