You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
140 lines
4.9 KiB
140 lines
4.9 KiB
'use strict';
|
|
const content = `--[[
|
|
Remove jobs from the specific set.
|
|
Input:
|
|
KEYS[1] set key,
|
|
KEYS[2] priority key
|
|
KEYS[3] rate limiter key
|
|
ARGV[1] prefix key
|
|
ARGV[2] maxTimestamp
|
|
ARGV[3] limit the number of jobs to be removed. 0 is unlimited
|
|
ARGV[4] set name, can be any of 'wait', 'active', 'paused', 'delayed', 'completed', or 'failed'
|
|
]]
|
|
local setKey = KEYS[1]
|
|
local priorityKey = KEYS[2]
|
|
local rateLimiterKey = KEYS[3]
|
|
local prefixKey = ARGV[1]
|
|
local maxTimestamp = ARGV[2]
|
|
local limitStr = ARGV[3]
|
|
local setName = ARGV[4]
|
|
local isList = false
|
|
local rcall = redis.call
|
|
-- Includes
|
|
--[[
|
|
Function to remove debounce key.
|
|
]]
|
|
local function removeDebounceKey(prefixKey, jobKey)
|
|
local debounceId = rcall("HGET", jobKey, "deid")
|
|
if debounceId then
|
|
local debounceKey = prefixKey .. "de:" .. debounceId
|
|
rcall("DEL", debounceKey)
|
|
end
|
|
end
|
|
if setName == "wait" or setName == "active" or setName == "paused" then
|
|
isList = true
|
|
end
|
|
-- We use ZRANGEBYSCORE to make the case where we're deleting a limited number
|
|
-- of items in a sorted set only run a single iteration. If we simply used
|
|
-- ZRANGE, we may take a long time traversing through jobs that are within the
|
|
-- grace period.
|
|
local function shouldUseZRangeByScore(isList, limit)
|
|
return not isList and limit > 0
|
|
end
|
|
local function getJobs(setKey, isList, rangeStart, rangeEnd, maxTimestamp, limit)
|
|
if isList then
|
|
return rcall("LRANGE", setKey, rangeStart, rangeEnd)
|
|
elseif shouldUseZRangeByScore(isList, limit) then
|
|
return rcall("ZRANGEBYSCORE", setKey, 0, maxTimestamp, "LIMIT", 0, limit)
|
|
else
|
|
return rcall("ZRANGE", setKey, rangeStart, rangeEnd)
|
|
end
|
|
end
|
|
local limit = tonumber(limitStr)
|
|
local rangeStart = 0
|
|
local rangeEnd = -1
|
|
-- If we're only deleting _n_ items, avoid retrieving all items
|
|
-- for faster performance
|
|
--
|
|
-- Start from the tail of the list, since that's where oldest elements
|
|
-- are generally added for FIFO lists
|
|
if limit > 0 then
|
|
rangeStart = -1 - limit + 1
|
|
rangeEnd = -1
|
|
end
|
|
local jobIds = getJobs(setKey, isList, rangeStart, rangeEnd, maxTimestamp, limit)
|
|
local deleted = {}
|
|
local deletedCount = 0
|
|
local jobTS
|
|
-- Run this loop:
|
|
-- - Once, if limit is -1 or 0
|
|
-- - As many times as needed if limit is positive
|
|
while ((limit <= 0 or deletedCount < limit) and next(jobIds, nil) ~= nil) do
|
|
local jobIdsLen = #jobIds
|
|
for i, jobId in ipairs(jobIds) do
|
|
if limit > 0 and deletedCount >= limit then
|
|
break
|
|
end
|
|
local jobKey = prefixKey .. jobId
|
|
if (rcall("EXISTS", jobKey .. ":lock") == 0) then
|
|
-- Find the right timestamp of the job to compare to maxTimestamp:
|
|
-- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
|
|
-- * processedOn represents when the job was last attempted, but it doesn't get populated until the job is first tried
|
|
-- * timestamp is the original job submission time
|
|
-- Fetch all three of these (in that order) and use the first one that is set so that we'll leave jobs that have been active within the grace period:
|
|
for _, ts in ipairs(rcall("HMGET", jobKey, "finishedOn", "processedOn", "timestamp")) do
|
|
if (ts) then
|
|
jobTS = ts
|
|
break
|
|
end
|
|
end
|
|
if (not jobTS or jobTS < maxTimestamp) then
|
|
if isList then
|
|
-- Job ids can't be the empty string. Use the empty string as a
|
|
-- deletion marker. The actual deletion will occur at the end of the
|
|
-- script.
|
|
rcall("LSET", setKey, rangeEnd - jobIdsLen + i, "")
|
|
else
|
|
rcall("ZREM", setKey, jobId)
|
|
end
|
|
rcall("ZREM", priorityKey, jobId)
|
|
if setName ~= "completed" and setName ~= "failed" then
|
|
removeDebounceKey(prefixKey, jobKey)
|
|
end
|
|
rcall("DEL", jobKey)
|
|
rcall("DEL", jobKey .. ":logs")
|
|
-- delete keys related to rate limiter
|
|
-- NOTE: this code is unncessary for other sets than wait, paused and delayed.
|
|
local limiterIndexTable = rateLimiterKey .. ":index"
|
|
local limitedSetKey = rcall("HGET", limiterIndexTable, jobId)
|
|
if limitedSetKey then
|
|
rcall("SREM", limitedSetKey, jobId)
|
|
rcall("HDEL", limiterIndexTable, jobId)
|
|
end
|
|
deletedCount = deletedCount + 1
|
|
table.insert(deleted, jobId)
|
|
end
|
|
end
|
|
end
|
|
-- If we didn't have a limit or used the single-iteration ZRANGEBYSCORE
|
|
-- function, return immediately. We should have deleted all the jobs we can
|
|
if limit <= 0 or shouldUseZRangeByScore(isList, limit) then
|
|
break
|
|
end
|
|
if deletedCount < limit then
|
|
-- We didn't delete enough. Look for more to delete
|
|
rangeStart = rangeStart - limit
|
|
rangeEnd = rangeEnd - limit
|
|
jobIds = getJobs(setKey, isList, rangeStart, rangeEnd, maxTimestamp, limit)
|
|
end
|
|
end
|
|
if isList then
|
|
rcall("LREM", setKey, 0, "")
|
|
end
|
|
return deleted
|
|
`;
|
|
module.exports = {
|
|
name: 'cleanJobsInSet',
|
|
content,
|
|
keys: 3,
|
|
};
|