You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

178 lines
5.9 KiB

'use strict';
const content = `--[[
Move job from active to a finished status (completed or failed)
A job can only be moved to completed if it was active.
The job must be locked before it can be moved to a finished status,
and the lock must be released in this script.
Input:
KEYS[1] active key
KEYS[2] completed/failed key
KEYS[3] jobId key
KEYS[4] wait key
KEYS[5] priority key
KEYS[6] active event key
KEYS[7] delayed key
KEYS[8] stalled key
KEYS[9] metrics key
ARGV[1] jobId
ARGV[2] timestamp
ARGV[3] msg property
ARGV[4] return value / failed reason
ARGV[5] token
ARGV[6] shouldRemove
ARGV[7] event data (? maybe just send jobid).
ARGV[8] should fetch next job
ARGV[9] base key
ARGV[10] lock token
ARGV[11] lock duration in milliseconds
ARGV[12] maxMetricsSize
Output:
0 OK
-1 Missing key.
-2 Missing lock.
-3 - Job not in active set.
Events:
'completed/failed'
]]
local rcall = redis.call
--[[
Functions to collect metrics based on a current and previous count of jobs.
Granualarity is fixed at 1 minute.
]]
local function collectMetrics(metaKey, dataPointsList, maxDataPoints, timestamp)
-- Increment current count
local count = rcall("HINCRBY", metaKey, "count", 1) - 1
-- Compute how many data points we need to add to the list, N.
local prevTS = rcall("HGET", metaKey, "prevTS")
if not prevTS then
-- If prevTS is nil, set it to the current timestamp
rcall("HSET", metaKey, "prevTS", timestamp, "prevCount", 0)
return
end
local N = math.floor((timestamp - prevTS) / 60000)
if N > 0 then
local delta = count - rcall("HGET", metaKey, "prevCount")
-- If N > 1, add N-1 zeros to the list
if N > 1 then
local points = {}
points[1] = delta
for i = 2, N do points[i] = 0 end
rcall("LPUSH", dataPointsList, unpack(points))
else
-- LPUSH delta to the list
rcall("LPUSH", dataPointsList, delta)
end
-- LTRIM to keep list to its max size
rcall("LTRIM", dataPointsList, 0, maxDataPoints - 1)
-- update prev count with current count
rcall("HSET", metaKey, "prevCount", count, "prevTS", timestamp)
end
end
-- Includes
local function removeLock(jobKey, stalledKey, token, jobId)
if token ~= "0" then
local lockKey = jobKey .. ':lock'
local lockToken = rcall("GET", lockKey)
if lockToken == token then
rcall("DEL", lockKey)
rcall("SREM", stalledKey, jobId)
else
if lockToken then
-- Lock exists but token does not match
return -6
else
-- Lock is missing completely
return -2
end
end
end
return 0
end
--[[
Function to remove debounce key if needed.
]]
local function removeDebounceKeyIfNeeded(prefixKey, debounceId)
if debounceId then
local debounceKey = prefixKey .. "de:" .. debounceId
local pttl = rcall("PTTL", debounceKey)
if pttl == 0 or pttl == -1 then
rcall("DEL", debounceKey)
end
end
end
if rcall("EXISTS", KEYS[3]) == 1 then -- // Make sure job exists
local errorCode = removeLock(KEYS[3], KEYS[8], ARGV[5], ARGV[1])
if errorCode < 0 then
return errorCode
end
-- Remove from active list (if not active we shall return error)
local numRemovedElements = rcall("LREM", KEYS[1], -1, ARGV[1])
if numRemovedElements < 1 then return -3 end
local debounceId = rcall("HGET", KEYS[3], "deid")
removeDebounceKeyIfNeeded(ARGV[9], debounceId)
-- Remove job?
local keepJobs = cmsgpack.unpack(ARGV[6])
local maxCount = keepJobs['count']
local maxAge = keepJobs['age']
local targetSet = KEYS[2]
local timestamp = ARGV[2]
if maxCount ~= 0 then
-- Add to complete/failed set
rcall("ZADD", targetSet, timestamp, ARGV[1])
rcall("HMSET", KEYS[3], ARGV[3], ARGV[4], "finishedOn", timestamp) -- "returnvalue" / "failedReason" and "finishedOn"
local function removeJobs(jobIds)
for i, jobId in ipairs(jobIds) do
local jobKey = ARGV[9] .. jobId
local jobLogKey = jobKey .. ':logs'
rcall("DEL", jobKey, jobLogKey)
end
end
-- Remove old jobs?
if maxAge ~= nil then
local start = timestamp - maxAge * 1000
local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf")
removeJobs(jobIds)
rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
end
if maxCount ~= nil and maxCount > 0 then
local start = maxCount
local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
removeJobs(jobIds)
rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1));
end
else
local jobLogKey = KEYS[3] .. ':logs'
rcall("DEL", KEYS[3], jobLogKey)
end
-- Collect metrics
if ARGV[12] ~= "" then
collectMetrics(KEYS[9], KEYS[9]..':data', ARGV[12], timestamp)
end
rcall("PUBLISH", targetSet, ARGV[7])
-- Try to get next job to avoid an extra roundtrip if the queue is not closing,
-- and not rate limited.
if (ARGV[8] == "1") then
-- move from wait to active
local jobId = rcall("RPOPLPUSH", KEYS[4], KEYS[1])
if jobId then
local jobKey = ARGV[9] .. jobId
local lockKey = jobKey .. ':lock'
-- get a lock
rcall("SET", lockKey, ARGV[11], "PX", ARGV[10])
rcall("ZREM", KEYS[5], jobId) -- remove from priority
rcall("PUBLISH", KEYS[6], jobId)
rcall("HSET", jobKey, "processedOn", ARGV[2])
return {rcall("HGETALL", jobKey), jobId} -- get job data
end
end
return 0
else
return -1
end
`;
module.exports = {
name: 'moveToFinished',
content,
keys: 9,
};