Bulgogi2: Created page with "local p = {} local parser = require('Module:WikitextParser') local yesno = require('Module:Yesno') local catmark = require('Module:CatMark') -- read assistance local deverrormsg = 'Categorizer config error: ' local function trim(value) if value == nil then return '' end return mw.text.trim(tostring(value)) end local function normalize(value, mode) local text = mw.ustring.lower(trim(value)) if mode == nil or mode == 'compact' then return (mw.ustring.gsub(text, '..."

2026-10-11T01:08:41Z

Created page with "local p = {} local parser = require('Module:WikitextParser') local yesno = require('Module:Yesno') local catmark = require('Module:CatMark') -- read assistance local deverrormsg = 'Categorizer config error: ' local function trim(value) if value == nil then return '' end return mw.text.trim(tostring(value)) end local function normalize(value, mode) local text = mw.ustring.lower(trim(value)) if mode == nil or mode == 'compact' then return (mw.ustring.gsub(text, '..."

New page

local p = {}
local parser = require('Module:WikitextParser')
local yesno = require('Module:Yesno')
local catmark = require('Module:CatMark') -- read assistance

local deverrormsg = 'Categorizer config error: '

local function trim(value)
if value == nil then return '' end
return mw.text.trim(tostring(value))
end

local function normalize(value, mode)
local text = mw.ustring.lower(trim(value))

if mode == nil or mode == 'compact' then
return (mw.ustring.gsub(text, '[^%w]', ''))
elseif mode == 'spaced' then
text = mw.ustring.gsub(text, '[^%w]+', ' ')
return trim(text)
end
error(deverrormsg .. 'Unknown normalization "' .. tostring(mode) .. '"', 0)
end

local function devError(message)
error(deverrormsg .. message, 0)
end

local function isDevError(err)
return type(err) == 'string' and string.sub(err, 1, #deverrormsg) == deverrormsg
end

local function need(tbl, key, where)
if tbl == nil then devError(where .. ' is missing') end
local value = tbl[key]
if value == nil then devError(where .. '.' .. key .. ' is missing') end
return value
end

local List = {}
List.__index = List

local function newList()
return setmetatable({ items = {}, seen = {} }, List)
end

function List:add(name)
name = trim(name)

if name ~= '' and not self.seen[name] then
self.seen[name] = true
self.items[#self.items + 1] = name
end
end

function List:list()
local copy = {}
for index, name in ipairs(self.items) do
copy[index] = name
end

return copy
end

local function fill(pattern, value)
if pattern == nil then return nil end
local replacement = trim(value)
return (mw.ustring.gsub(pattern, '%$1', function()
return replacement
end))
end

local fileNamespace = 6
local categoryNamespace = 14

local function replacePlain(text, needle, replacement)
if needle == '' then return text end
local parts = {}
local position = 1

while true do
local start, finish = string.find(text, needle, position, true)
if not start then break end
parts[#parts + 1] = string.sub(text, position, start - 1)
parts[#parts + 1] = replacement
position = finish + 1
end
parts[#parts + 1] = string.sub(text, position)
return table.concat(parts)
end

local function stripHidden(value)
local text = tostring(value or '')

for _, reference in ipairs(parser.getReferences(text)) do
text = replacePlain(text, reference, ' ')
end
return mw.text.killMarkers(text)
end

local function linkDisplayText(link)
local inner = string.match(link, '^%[%[(.*)%]%]$')
if not inner then return link end
local target, label = string.match(inner, '^([^|]*)|(.*)$')
if not target then target = inner end
local visibleLink = string.match(target, '^%s*:') ~= nil
local titleText = mw.ustring.gsub(target, '^%s*:%s*', '')
titleText = mw.text.split(titleText, '#', true)[1]
local title = mw.title.new(trim(titleText))
if title and not visibleLink and (title.namespace == fileNamespace or title.namespace == categoryNamespace) then
return ''
end
if label and trim(label) ~= '' then return label end

return (mw.ustring.gsub(target, '^%s*:%s*', ''))
end

local function clean(value)
local text = stripHidden(value)

for _, link in ipairs(parser.getLinks(text)) do
text = replacePlain(text, link, linkDisplayText(link))
end

text = mw.ustring.gsub(text, '%[%s*(%a*:?//[^%s%]]*)([^%]]*)%]', function(url, label)
local scheme = string.lower(string.match(url, '^(%a*:?)//') or '')
if scheme == 'http:' or scheme == 'https:' or scheme == '' then return ' ' .. label .. ' ' end
end)

text = mw.ustring.gsub(text, '(%a+)://[^%s<>%[%]"]+', function(scheme)
scheme = string.lower(scheme)
if scheme == 'http' or scheme == 'https' then return ' ' end
end)

text = mw.ustring.gsub(text, '<!%-%-.-%-%->', ' ')
text = mw.ustring.gsub(text, '</?%a[^<>]*>', ' ')
text = mw.text.decode(text, true)

return trim(text)
end

local function findYears(value)
local text = clean(value)
local years = {}
local seen = {}

for year in mw.ustring.gmatch(text, '%f[%d](%d%d%d%d)%f[%D]') do
if not seen[year] then
seen[year] = true
years[#years + 1] = year
end
end
return years
end

local function findYearRanges(value) -- to handle statements like 2022 - 2026, another function is needed, with interpretation left to the submodules
local text = clean(value)
local ranges = {}
local seen = {}

for first, second in mw.ustring.gmatch(text, '%f[%d](%d%d%d%d)%s*[%-–—][^%-–—]-%f[%d](%d%d%d%d)%f[%D]') do
local key = first .. '-' .. second

if not seen[key] then
seen[key] = true
ranges[#ranges + 1] = { first, second }
end
end
return ranges -- return all ranges, regardless of how many. active in 2021 - 2022, 2025 - 2026, return all, interpret at submodule level
end

local function findLinkTargets(value, opts)
local namespaces = {}

for _, namespace in ipairs((opts and opts.namespaces) or { 0 }) do
namespaces[namespace] = true
end

local text = stripHidden(value)
local targets = {}
local seen = {}

for _, link in ipairs(parser.getLinks(text)) do
local inner = string.match(link, '^%[%[(.*)%]%]$')

if inner then
local target = mw.text.split(inner, '|', true)[1]
target = mw.text.split(target, '#', true)[1]
target = trim((mw.ustring.gsub(target, '^%s*:%s*', '')))

if target ~= '' then
local title = mw.title.new(target)

if title and not title.isExternal and namespaces[title.namespace] then
local name = title.prefixedText

if not seen[name] then
seen[name] = true
targets[#targets + 1] = name
end
end
end
end
end
return targets
end

local function findDomains(value)
local text = stripHidden(value)
local hosts = {}
local seen = {}

for start, scheme, host in string.gmatch(text, '()(%a*:?)//([%w%.%-]+)') do -- no need for escaping forward slash, literal
local loweredScheme = string.lower(scheme)
local isUrl = loweredScheme == 'http:'
or loweredScheme == 'https:'
or (scheme == '' and string.sub(text, start - 1, start - 1) == '[')

if isUrl then -- raw hosts only, MAS maps them to sites via domains list, redirect hosts are handled that way
host = string.lower(host)
host = string.gsub(host, '^www%.', '')
host = string.gsub(host, '%.+$', '')

if host ~= '' and not seen[host] then
seen[host] = true
hosts[#hosts + 1] = host
end
end
end
return hosts
end

local dataPrefix = catmark.prefix

local function readDataAttr(value, name)
if type(name) ~= 'string' or not string.match(name, '^[%w%-]+$') then
devError(
'RDA: Attribute names from CatMark need to be letters, digits, and/or hyphens. Got: ' .. tostring(name)
)
end

local text = stripHidden(value)
local attr = dataPrefix .. string.lower(name)
local pattern = string.gsub(attr, '%-', '%%-') .. '="([^"]*)"'
local values = {}

for found in string.gmatch(text, pattern) do
values[#values + 1] = trim(mw.text.decode(found, true))
end
return values
end

local function parseNumber(value)
if value == nil then return nil end

local cleaned = string.gsub(trim(value), '[,%s]', '')
if not string.match(cleaned, '^%-?%d+%.?%d*$') then return nil end
return tonumber(cleaned)
end

local function pickBand(number, bands) -- bands = number ranges for categories, i.e. the system to handle 100-999 soybooru posts is resolved here
if type(bands) ~= 'table' then devError('pickBand: expected a list of bands, got: ' .. type(bands)) end
if type(number) ~= 'number' then return nil end

local best

for index, band in ipairs(bands) do
if type(band.min) ~= 'number' or type(band.name) ~= 'string' then
devError('pickBand: band ' .. index .. ' needs a numeric min and a string name')
end

if number >= band.min and (best == nil or band.min > best.min) then best = band end
end
return best and best.name or nil
end

local matchModes = { word = true, substring = true, exact = true }
local function prepare(text, mode, caseSensitive)
local prepared = trim(text)

if not caseSensitive then prepared = mw.ustring.lower(prepared) end

if mode == 'substring' then return (mw.ustring.gsub(prepared, '[^%w]', '')) end
prepared = mw.ustring.gsub(prepared, '[^%w]+', ' ')
return trim(prepared)
end

local function normalizeDomain(domain)
domain = string.lower(trim(domain))
domain = string.gsub(domain, '^www%.', '')
domain = string.gsub(domain, '%.+$', '')
return domain
end

local function compileAliasSet(sets, opts)
local defaultMode = (opts and opts.match) or 'word'
local compiled = { entries = {} }
local byCategory = {}
local claimedAliases = {}
local claimedDomains = {}

if type(sets) ~= 'table' then devError('CAS: expected a list of alias sets, got this:' .. type(sets)) end
if type(sets[1]) == 'table' and sets[1].category ~= nil then
devError('CAS: got a single alias set instead of a list. Did you forget to wrap?')
end
for setIndex = 1, table.maxn(sets) do
local set = sets[setIndex]
if type(set) ~= 'table' then
devError(
'CAS: alias set '
.. setIndex
.. ' is '
.. type(set)
.. '. Missing config key? Use config.sites or {} to fix'
)
end
local setMode = set.match or defaultMode
local setAliasOwners = {}
local setDomainOwners = {}

for entryIndex, entry in ipairs(set) do
local where = 'alias set ' .. setIndex .. ', entry ' .. entryIndex
local category = need(entry, 'category', where)

if type(category) ~= 'string' or trim(category) == '' then
devError(where .. ': category must be a noempty string')
end

category = trim(category)

if entry.aliases ~= nil and type(entry.aliases) ~= 'table' then
devError(where .. ': aliases must be a list')
end
if entry.domains ~= nil and type(entry.domains) ~= 'table' then
devError(where .. ': domains must be a list')
end

local entryMode = entry.match or setMode
local target = byCategory[category]

if not target then
target = { category = category, aliases = {}, domains = {} }
byCategory[category] = target
compiled.entries[#compiled.entries + 1] = target
end

for aliasIndex, alias in ipairs(entry.aliases or {}) do
local text, mode, caseSensitive

if type(alias) == 'string' then
text, mode, caseSensitive = alias, entryMode, false
elseif type(alias) == 'table' and type(alias[1]) == 'string' then
text = alias[1]
mode = alias.match or entryMode
caseSensitive = alias.caseSensitive == true
else
devError(where .. ', alias ' .. aliasIndex .. ': must be a string or { text, ... }')
end
if not matchModes[mode] then
devError(where .. ', alias' .. aliasIndex .. ': unknown match mode"' .. tostring(mode) .. '"')
end

local key = prepare(text, mode, caseSensitive)

if key ~= '' then
local claimKey = mode .. '|' .. (caseSensitive and 'c' or 'i') .. '|' .. key
local setOwner = setAliasOwners[claimKey]

if setOwner ~= nil and setOwner ~= category then
devError(where .. ', alias"' .. text .. '": already used by"' .. setOwner .. '"in the same set')
end
setAliasOwners[claimKey] = category
if not claimedAliases[claimKey] then
claimedAliases[claimKey] = true
target.aliases[#target.aliases + 1] = {
key = key,
mode = mode,
caseSensitive = caseSensitive,
}
end
end
end

for domainIndex, domain in ipairs(entry.domains or {}) do
if type(domain) ~= 'string' then
devError(where .. ', domain ' .. domainIndex .. ': must be a string')
end

domain = normalizeDomain(domain)

if domain ~= '' then
local setOwner = setDomainOwners[domain]

if setOwner ~= nil and setOwner ~= category then
devError(
where .. ', domain"' .. domain .. '": already used by"' .. setOwner .. '"in the same set'
)
end
setDomainOwners[domain] = category

if not claimedDomains[domain] then
claimedDomains[domain] = true
target.domains[#target.domains + 1] = domain
end
end
end
end
end
return compiled
end

local function domainMatches(host, domain)
if host == domain then return true end

local suffix = '.' .. domain
return #host > #suffix and string.sub(host, -#suffix) == suffix
end

local function matchAliasSets(value, compiled)
if compiled == nil then
devError('MAS: got nil instead of a set. Is the variable holding the result of CAS misspelled or not defined?')
elseif type(compiled) ~= 'table' or type(compiled.entries) ~= 'table' then
if type(compiled) == 'table' and type(compiled[1]) == 'table' and compiled[1].category ~= nil then -- then im illiterate
devError(
'MAS: got raw alias set from config. Compile it first local sites = core.compileAliasSet({ config.sites }), then pass sites'
)
end
devError('MAS: expected an alias set compiled by CAS. got whatever this is:' .. type(compiled))
end
local text = clean(value)
local preparedTexts = {}
local hosts

local function preparedText(mode, caseSensitive)
local cacheKey = mode .. (caseSensitive and 'c' or 'i')

if preparedTexts[cacheKey] == nil then preparedTexts[cacheKey] = prepare(text, mode, caseSensitive) end

return preparedTexts[cacheKey]
end

local results = {}

for _, entry in ipairs(compiled.entries) do
local matched = false

for _, alias in ipairs(entry.aliases) do
local candidate = preparedText(alias.mode, alias.caseSensitive)

if alias.mode == 'substring' then
matched = string.find(candidate, alias.key, 1, true) ~= nil
elseif alias.mode == 'word' then
matched = string.find(' ' .. candidate .. ' ', ' ' .. alias.key .. ' ', 1, true) ~= nil
else
matched = candidate == alias.key
end

if matched then break end
end
if not matched and #entry.domains > 0 then
hosts = hosts or findDomains(value)

for _, host in ipairs(hosts) do
for _, domain in ipairs(entry.domains) do
if domainMatches(host, domain) then
matched = true
break
end
end
if matched then break end
end
end
if matched then results[#results + 1] = entry.category end
end

return results
end

local function parseYearOverride(value)
local cleaned = trim(value)

if normalize(cleaned) == 'undated' then
return {}, true
elseif mw.ustring.match(cleaned, '^%d%d%d%d$') then
return { cleaned }, true
end

return {}, false
end

local function overrideLookup(compiled)
if compiled.overrideLookup then return compiled.overrideLookup end

local lookup = {}

for _, entry in ipairs(compiled.entries) do
local categoryKey = normalize(entry.category)

if categoryKey ~= '' and lookup[categoryKey] == nil then lookup[categoryKey] = entry.category end

for _, alias in ipairs(entry.aliases) do
local aliasKey = mw.ustring.lower((string.gsub(alias.key, ' ', '')))

if aliasKey ~= '' and lookup[aliasKey] == nil then lookup[aliasKey] = entry.category end
end
end
compiled.overrideLookup = lookup
return lookup
end

local function parseAliasSetOverride(value, compiled) -- parse site override 2.0
if type(compiled) ~= 'table' or type(compiled.entries) ~= 'table' then
devError('PASO: expected alias set compiled by CAS, got: ' .. type(compiled))
end

local cleaned = trim(value)

if normalize(cleaned) == 'none' then return {}, true end

local lookup = overrideLookup(compiled)
local values = {}
local seen = {}

for _, piece in ipairs(mw.text.split(cleaned, ',', true)) do
local key = normalize(piece)
local category = lookup[key]

if key == '' or category == nil then return {}, false end

if not seen[category] then
seen[category] = true
values[#values + 1] = category
end
end

return values, true
end

local function parseLinkTargetOverride(value, opts)
local namespaces = {}

for _, namespace in ipairs((opts and opts.namespaces) or { 0 }) do
namespaces[namespace] = true
end

local target = trim(value)
local inner = string.match(target, '^%[%[(.*)%]%]$')

if inner then
if string.find(inner, '[%[%]]') then return {}, false end

target = mw.text.split(inner, '|', true)[1]
end

target = mw.text.split(target, '#', true)[1]
target = trim((mw.ustring.gsub(target, '^%s*:%s*', '')))

if target == '' then return {}, false end

local title = mw.title.new(target)
if title and not title.isExternal and namespaces[title.namespace] then return { title.prefixedText }, true end

return {}, false
end

local function parseExactOverride(value, map)
if type(map) ~= 'table' then devError('PEO: expected a map of accepted values, got: ' .. type(map)) end

local lookup = {}

for key, mapped in pairs(map) do
if type(key) == 'number' then key = mapped end

if type(key) ~= 'string' or type(mapped) ~= 'string' then
devError('PEO: map keys and values must be strings')
end

local normalizedKey = normalize(key)

if lookup[normalizedKey] ~= nil and lookup[normalizedKey] ~= mapped then
devError('PEO: "' .. key .. '" collides with another key after normalization, remove it')
end

lookup[normalizedKey] = mapped
end

local mapped = lookup[normalize(value)]
if trim(value) ~= '' and mapped ~= nil then return { mapped }, true end

return {}, false
end

-- // compiling complete, resolvers //
local function processValues(values, canon, filter)
local result = {}
local seen = {}
local rejected = false

for _, value in ipairs(values) do
local processed = value

if canon then processed = canon(processed) end

processed = processed ~= nil and trim(processed) or ''

if processed == '' or (filter and not filter(processed)) then
rejected = true
elseif not seen[processed] then
seen[processed] = true
result[#result + 1] = processed
end
end
return result, rejected
end

local function resolve(spec)
if type(spec) ~= 'table' then devError('resolver: expected table with named arguments, got: ' .. type(spec)) end

local parse = spec.parse
local canon = spec.canon
local filter = spec.filter
local max = spec.max

if parse ~= nil and type(parse) ~= 'function' then
devError('resolver: parse must be a function, got: ' .. type(parse))
end
if canon ~= nil and type(canon) ~= 'function' then
devError('resolver: canon must be a function, got: ' .. type(canon))
end
if filter ~= nil and type(filter) ~= 'function' then
devError('resolver: filter must be a function, got: ' .. type(filter))
end
if max ~= nil and (type(max) ~= 'number' or max < 1 or max % 1 ~= 0) then -- necessary check to avoid hard braps
devError('resolver: MATH! MAX must be positive integer, got ' .. tostring(max))
end

local state = {
values = {},
source = 'detected',
overrideProvided = trim(spec.override) ~= '',
invalid = false,
undetected = false,
overflow = false,
customized = false,
}

if spec.applicable == false then
state.source = 'inapplicable'
state.invalid = state.overrideProvided
state.customized = state.overrideProvided or yesno(spec.suppress, false) == true
return state
end

if yesno(spec.suppress, false) then
state.source = 'suppressed'
state.invalid = state.overrideProvided
state.customized = true
return state
end

state.customized = state.overrideProvided

if state.overrideProvided then
if parse == nil then devError('resolver: You provided an override, but did not provide a parse function') end

local values, valid = parse(spec.override)
if valid then
if type(values) ~= 'table' then
devError('resolver: parse must return a list of values, got ' .. type(values))
end

local processed, rejected = processValues(values, canon, filter)
if not rejected and (max == nil or #processed <= max) then
state.source = 'override'
state.values = processed
return state
end
end

state.invalid = true
end

local candidates = spec.candidates or {}

if type(candidates) ~= 'table' then devError('resolver: candidates must be a list, got ' .. type(candidates)) end

local processed = processValues(candidates, canon, filter)
if max and #processed > max then
state.overflow = true
for index = #processed, max + 1, -1 do
processed[index] = nil
end
end

state.values = processed
state.undetected = #processed == 0
return state
end

local function anyFlag(flag, ...)
for index = 1, select('#', ...) do
local state = select(index, ...)
if type(state) == 'table' and state[flag] then return true end
end
return false
end

local function anyCustomized(...)
return anyFlag('customized', ...)
end

local function anyInvalid(...)
return anyFlag('invalid', ...)
end

local function fromMap(map, opts)
if type(map) ~= 'table' then
devError('fromMap: expected a table, got ' .. type(map) .. 'something seriously went wrong.')
end

local strict = opts ~= nil and opts.strict == true
return function(value)
local mapped = map[value]
if mapped ~= nil then
return mapped
elseif strict then
return nil
end

return value
end
end

local errorCat = 'Pages with categorizer errors'
local missingCat = 'Pages with nonexistent categorizer categories' -- module is trying to call a redlink cat, possibly requiring admin intervention at alias data

local function makeMain(categoriesFn, namespaces)
if type(categoriesFn) ~= 'function' then
devError('M&Ms: expected the subject categories function, got ' .. type(categoriesFn))
end

local allowed = {}

for _, namespace in ipairs(namespaces or { 0 }) do
allowed[namespace] = true
end

return function(frame)
local parent = frame:getParent()
if not parent then return '' end
local args = parent.args
local title = mw.title.getCurrentTitle()

if not allowed[title.namespace] or yesno(args.nocat, false) then return '' end

local ok, result = pcall(categoriesFn, args)

if not ok then -- quiet down non critical crashes, issue cat
if isDevError(result) then error(result, 0) end
mw.addWarning('Categorizer error: ' .. tostring(result))
return '[[Category:' .. errorCat .. ']]'
end

if getmetatable(result) == List then result = result:list() end

if type(result) ~= 'table' then
devError('M&Ms: the categories function must return a list, but got: ' .. type(result))
end

local output = {}
local missing = {}
local failed = false

for _, name in ipairs(result) do
name = trim(name)

if name ~= '' then
local title = mw.title.new(name, categoryNamespace)

if title then
output[#output + 1] = '[[Category:' .. name .. ']]'
local ok, exists = pcall(function()
return title.exists -- expensive call, massive value
end)
if ok and not exists then missing[#missing + 1] = '[[:Category:' .. name .. ']]' end -- instead of silently spamming red cats, also call an administrator/maintenance editor to review & handle the new red cats
else
failed = true
end
end
end
if failed then output[#output + 1] = '[[Category:' .. errorCat .. ']]' end
if #missing > 0 then
output[#output + 1] = '[[Category:' .. missingCat .. ']]'
mw.addWarning(
'Categorizer added nonexistent categories. If needed, create them, otherwise check subject configs or alias lists for fix. Added: '
.. table.concat(missing, ', ')
)
end

return table.concat(output)
end
end

p.trim = trim
p.normalize = normalize
p.devError = devError
p.need = need
p.newList = newList
p.fill = fill
p.clean = clean

p.findYears = findYears
p.findYearRanges = findYearRanges
p.findLinkTargets = findLinkTargets
p.readDataAttr = readDataAttr
p.parseNumber = parseNumber
p.pickBand = pickBand
p.compileAliasSet = compileAliasSet
p.matchAliasSets = matchAliasSets

p.parseYearOverride = parseYearOverride
p.parseAliasSetOverride = parseAliasSetOverride
p.parseLinkTargetOverride = parseLinkTargetOverride
p.parseExactOverride = parseExactOverride

p.resolve = resolve
p.anyCustomized = anyCustomized
p.anyInvalid = anyInvalid
p.fromMap = fromMap
p.makeMain = makeMain

return p

Module:Infobox categorizer - Revision history