Editing Module:Infobox categorizer

local p = {}
local parser = require('Module:WikitextParser')
local yesno = require('Module:Yesno')
local catmark = require('Module:CatMark') -- read assistance

local deverrormsg = 'Categorizer config error: '

local function trim(value)
	if value == nil then return '' end
	return mw.text.trim(tostring(value))
end

local function normalize(value, mode)
	local text = mw.ustring.lower(trim(value))

	if mode == nil or mode == 'compact' then
		return (mw.ustring.gsub(text, '[^%w]', ''))
	elseif mode == 'spaced' then
		text = mw.ustring.gsub(text, '[^%w]+', ' ')
		return trim(text)
	end
	error(deverrormsg .. 'Unknown normalization "' .. tostring(mode) .. '"', 0)
end

local function devError(message)
	error(deverrormsg .. message, 0)
end

local function isDevError(err)
	return type(err) == 'string' and string.sub(err, 1, #deverrormsg) == deverrormsg
end

local function need(tbl, key, where)
	if tbl == nil then devError(where .. ' is missing') end
	local value = tbl[key]
	if value == nil then devError(where .. '.' .. key .. ' is missing') end
	return value
end

local List = {}
List.__index = List

local function newList()
	return setmetatable({ items = {}, seen = {} }, List)
end

function List:add(name)
	name = trim(name)

	if name ~= '' and not self.seen[name] then
		self.seen[name] = true
		self.items[#self.items + 1] = name
	end
end

function List:list()
	local copy = {}
	for index, name in ipairs(self.items) do
		copy[index] = name
	end

	return copy
end

local function fill(pattern, value)
	if pattern == nil then return nil end
	local replacement = trim(value)
	return (mw.ustring.gsub(pattern, '%$1', function()
		return replacement
	end))
end

local fileNamespace = 6
local categoryNamespace = 14

local function replacePlain(text, needle, replacement)
	if needle == '' then return text end
	local parts = {}
	local position = 1

	while true do
		local start, finish = string.find(text, needle, position, true)
		if not start then break end
		parts[#parts + 1] = string.sub(text, position, start - 1)
		parts[#parts + 1] = replacement
		position = finish + 1
	end
	parts[#parts + 1] = string.sub(text, position)
	return table.concat(parts)
end

local function stripHidden(value)
	local text = tostring(value or '')

	for _, reference in ipairs(parser.getReferences(text)) do
		text = replacePlain(text, reference, ' ')
	end
	return mw.text.killMarkers(text)
end

local function linkDisplayText(link)
	local inner = string.match(link, '^%[%[(.*)%]%]$')
	if not inner then return link end
	local target, label = string.match(inner, '^([^|]*)|(.*)$')
	if not target then target = inner end
	local visibleLink = string.match(target, '^%s*:') ~= nil
	local titleText = mw.ustring.gsub(target, '^%s*:%s*', '')
	titleText = mw.text.split(titleText, '#', true)[1]
	local title = mw.title.new(trim(titleText))
	if title and not visibleLink and (title.namespace == fileNamespace or title.namespace == categoryNamespace) then
		return ''
	end
	if label and trim(label) ~= '' then return label end

	return (mw.ustring.gsub(target, '^%s*:%s*', ''))
end

local function clean(value)
	local text = stripHidden(value)

	for _, link in ipairs(parser.getLinks(text)) do
		text = replacePlain(text, link, linkDisplayText(link))
	end

	text = mw.ustring.gsub(text, '%[%s*(%a*:?//[^%s%]]*)([^%]]*)%]', function(url, label)
		local scheme = string.lower(string.match(url, '^(%a*:?)//') or '')
		if scheme == 'http:' or scheme == 'https:' or scheme == '' then return ' ' .. label .. ' ' end
	end)

	text = mw.ustring.gsub(text, '(%a+)://[^%s<>%[%]"]+', function(scheme)
		scheme = string.lower(scheme)
		if scheme == 'http' or scheme == 'https' then return ' ' end
	end)

	text = mw.ustring.gsub(text, '<!%-%-.-%-%->', ' ')
	text = mw.ustring.gsub(text, '</?%a[^<>]*>', ' ')
	text = mw.text.decode(text, true)

	return trim(text)
end

local function findYears(value)
	local text = clean(value)
	local years = {}
	local seen = {}

	for year in mw.ustring.gmatch(text, '%f[%d](%d%d%d%d)%f[%D]') do
		if not seen[year] then
			seen[year] = true
			years[#years + 1] = year
		end
	end
	return years
end

local function findYearRanges(value) -- to handle statements like 2022 - 2026, another function is needed, with interpretation left to the submodules
	local text = clean(value)
	local ranges = {}
	local seen = {}

	for first, second in mw.ustring.gmatch(text, '%f[%d](%d%d%d%d)%s*[%-–—][^%-–—]-%f[%d](%d%d%d%d)%f[%D]') do
		local key = first .. '-' .. second

		if not seen[key] then
			seen[key] = true
			ranges[#ranges + 1] = { first, second }
		end
	end
	return ranges -- return all ranges, regardless of how many. active in 2021 - 2022, 2025 - 2026, return all, interpret at submodule level
end

local function findLinkTargets(value, opts)
	local namespaces = {}

	for _, namespace in ipairs((opts and opts.namespaces) or { 0 }) do
		namespaces[namespace] = true
	end

	local text = stripHidden(value)
	local targets = {}
	local seen = {}

	for _, link in ipairs(parser.getLinks(text)) do
		local inner = string.match(link, '^%[%[(.*)%]%]$')

		if inner then
			local target = mw.text.split(inner, '|', true)[1]
			target = mw.text.split(target, '#', true)[1]
			target = trim((mw.ustring.gsub(target, '^%s*:%s*', '')))

			if target ~= '' then
				local title = mw.title.new(target)

				if title and not title.isExternal and namespaces[title.namespace] then
					local name = title.prefixedText

					if not seen[name] then
						seen[name] = true
						targets[#targets + 1] = name
					end
				end
			end
		end
	end
	return targets
end

local function findDomains(value)
	local text = stripHidden(value)
	local hosts = {}
	local seen = {}

	for start, scheme, host in string.gmatch(text, '()(%a*:?)//([%w%.%-]+)') do -- no need for escaping forward slash, literal
		local loweredScheme = string.lower(scheme)
		local isUrl = loweredScheme == 'http:'
			or loweredScheme == 'https:'
			or (scheme == '' and string.sub(text, start - 1, start - 1) == '[')

		if isUrl then -- raw hosts only, MAS maps them to sites via domains list, redirect hosts are handled that way
			host = string.lower(host)
			host = string.gsub(host, '^www%.', '')
			host = string.gsub(host, '%.+$', '')

			if host ~= '' and not seen[host] then
				seen[host] = true
				hosts[#hosts + 1] = host
			end
		end
	end
	return hosts
end

local dataPrefix = catmark.prefix

local function readDataAttr(value, name)
	if type(name) ~= 'string' or not string.match(name, '^[%w%-]+$') then
		devError(
			'RDA: Attribute names from CatMark need to be letters, digits, and/or hyphens. Got: ' .. tostring(name)
		)
	end

	local text = stripHidden(value)
	local attr = dataPrefix .. string.lower(name)
	local pattern = string.gsub(attr, '%-', '%%-') .. '="([^"]*)"'
	local values = {}

	for found in string.gmatch(text, pattern) do
		values[#values + 1] = trim(mw.text.decode(found, true))
	end
	return values
end

local function parseNumber(value)
	if value == nil then return nil end

	local cleaned = string.gsub(trim(value), '[,%s]', '')
	if not string.match(cleaned, '^%-?%d+%.?%d*$') then return nil end
	return tonumber(cleaned)
end

local function pickBand(number, bands) -- bands = number ranges for categories, i.e. the system to handle 100-999 soybooru posts is resolved here
	if type(bands) ~= 'table' then devError('pickBand: expected a list of bands, got: ' .. type(bands)) end
	if type(number) ~= 'number' then return nil end

	local best

	for index, band in ipairs(bands) do
		if type(band.min) ~= 'number' or type(band.name) ~= 'string' then
			devError('pickBand: band ' .. index .. ' needs a numeric min and a string name')
		end

		if number >= band.min and (best == nil or band.min > best.min) then best = band end
	end
	return best and best.name or nil
end

local matchModes = { word = true, substring = true, exact = true }
local function prepare(text, mode, caseSensitive)
	local prepared = trim(text)

	if not caseSensitive then prepared = mw.ustring.lower(prepared) end

	if mode == 'substring' then return (mw.ustring.gsub(prepared, '[^%w]', '')) end
	prepared = mw.ustring.gsub(prepared, '[^%w]+', ' ')
	return trim(prepared)
end

local function normalizeDomain(domain)
	domain = string.lower(trim(domain))
	domain = string.gsub(domain, '^www%.', '')
	domain = string.gsub(domain, '%.+$', '')
	return domain
end

local function compileAliasSet(sets, opts)
	local defaultMode = (opts and opts.match) or 'word'
	local compiled = { entries = {} }
	local byCategory = {}
	local claimedAliases = {}
	local claimedDomains = {}

	if type(sets) ~= 'table' then devError('CAS: expected a list of alias sets, got this:' .. type(sets)) end
	if type(sets[1]) == 'table' and sets[1].category ~= nil then
		devError('CAS: got a single alias set instead of a list. Did you forget to wrap?')
	end
	for setIndex = 1, table.maxn(sets) do
		local set = sets[setIndex]
		if type(set) ~= 'table' then
			devError(
				'CAS: alias set '
					.. setIndex
					.. ' is '
					.. type(set)
					.. '. Missing config key? Use config.sites or {} to fix'
			)
		end
		local setMode = set.match or defaultMode
		local setAliasOwners = {}
		local setDomainOwners = {}

		for entryIndex, entry in ipairs(set) do
			local where = 'alias set ' .. setIndex .. ', entry ' .. entryIndex
			local category = need(entry, 'category', where)

			if type(category) ~= 'string' or trim(category) == '' then
				devError(where .. ': category must be a noempty string')
			end

			category = trim(category)

			if entry.aliases ~= nil and type(entry.aliases) ~= 'table' then
				devError(where .. ': aliases must be a list')
			end
			if entry.domains ~= nil and type(entry.domains) ~= 'table' then
				devError(where .. ': domains must be a list')
			end

			local entryMode = entry.match or setMode
			local target = byCategory[category]

			if not target then
				target = { category = category, aliases = {}, domains = {} }
				byCategory[category] = target
				compiled.entries[#compiled.entries + 1] = target
			end

			for aliasIndex, alias in ipairs(entry.aliases or {}) do
				local text, mode, caseSensitive

				if type(alias) == 'string' then
					text, mode, caseSensitive = alias, entryMode, false
				elseif type(alias) == 'table' and type(alias[1]) == 'string' then
					text = alias[1]
					mode = alias.match or entryMode
					caseSensitive = alias.caseSensitive == true
				else
					devError(where .. ', alias ' .. aliasIndex .. ': must be a string or { text, ... }')
				end
				if not matchModes[mode] then
					devError(where .. ', alias' .. aliasIndex .. ': unknown match mode"' .. tostring(mode) .. '"')
				end

				local key = prepare(text, mode, caseSensitive)

				if key ~= '' then
					local claimKey = mode .. '|' .. (caseSensitive and 'c' or 'i') .. '|' .. key
					local setOwner = setAliasOwners[claimKey]

					if setOwner ~= nil and setOwner ~= category then
						devError(where .. ', alias"' .. text .. '": already used by"' .. setOwner .. '"in the same set')
					end
					setAliasOwners[claimKey] = category
					if not claimedAliases[claimKey] then
						claimedAliases[claimKey] = true
						target.aliases[#target.aliases + 1] = {
							key = key,
							mode = mode,
							caseSensitive = caseSensitive,
						}
					end
				end
			end

			for domainIndex, domain in ipairs(entry.domains or {}) do
				if type(domain) ~= 'string' then
					devError(where .. ', domain ' .. domainIndex .. ': must be a string')
				end

				domain = normalizeDomain(domain)

				if domain ~= '' then
					local setOwner = setDomainOwners[domain]

					if setOwner ~= nil and setOwner ~= category then
						devError(
							where .. ', domain"' .. domain .. '": already used by"' .. setOwner .. '"in the same set'
						)
					end
					setDomainOwners[domain] = category

					if not claimedDomains[domain] then
						claimedDomains[domain] = true
						target.domains[#target.domains + 1] = domain
					end
				end
			end
		end
	end
	return compiled
end

local function domainMatches(host, domain)
	if host == domain then return true end

	local suffix = '.' .. domain
	return #host > #suffix and string.sub(host, -#suffix) == suffix
end

local function matchAliasSets(value, compiled)
	if compiled == nil then
		devError('MAS: got nil instead of a set. Is the variable holding the result of CAS misspelled or not defined?')
	elseif type(compiled) ~= 'table' or type(compiled.entries) ~= 'table' then
		if type(compiled) == 'table' and type(compiled[1]) == 'table' and compiled[1].category ~= nil then -- then im illiterate
			devError(
				'MAS: got raw alias set from config. Compile it first local sites = core.compileAliasSet({ config.sites }), then pass sites'
			)
		end
		devError('MAS: expected an alias set compiled by CAS. got whatever this is:' .. type(compiled))
	end
	local text = clean(value)
	local preparedTexts = {}
	local hosts

	local function preparedText(mode, caseSensitive)
		local cacheKey = mode .. (caseSensitive and 'c' or 'i')

		if preparedTexts[cacheKey] == nil then preparedTexts[cacheKey] = prepare(text, mode, caseSensitive) end

		return preparedTexts[cacheKey]
	end

	local results = {}

	for _, entry in ipairs(compiled.entries) do
		local matched = false

		for _, alias in ipairs(entry.aliases) do
			local candidate = preparedText(alias.mode, alias.caseSensitive)

			if alias.mode == 'substring' then
				matched = string.find(candidate, alias.key, 1, true) ~= nil
			elseif alias.mode == 'word' then
				matched = string.find(' ' .. candidate .. ' ', ' ' .. alias.key .. ' ', 1, true) ~= nil
			else
				matched = candidate == alias.key
			end

			if matched then break end
		end
		if not matched and #entry.domains > 0 then
			hosts = hosts or findDomains(value)

			for _, host in ipairs(hosts) do
				for _, domain in ipairs(entry.domains) do
					if domainMatches(host, domain) then
						matched = true
						break
					end
				end
				if matched then break end
			end
		end
		if matched then results[#results + 1] = entry.category end
	end

	return results
end

local function parseYearOverride(value)
	local cleaned = trim(value)

	if normalize(cleaned) == 'undated' then
		return {}, true
	elseif mw.ustring.match(cleaned, '^%d%d%d%d$') then
		return { cleaned }, true
	end

	return {}, false
end

local function overrideLookup(compiled)
	if compiled.overrideLookup then return compiled.overrideLookup end

	local lookup = {}

	for _, entry in ipairs(compiled.entries) do
		local categoryKey = normalize(entry.category)

		if categoryKey ~= '' and lookup[categoryKey] == nil then lookup[categoryKey] = entry.category end

		for _, alias in ipairs(entry.aliases) do
			local aliasKey = mw.ustring.lower((string.gsub(alias.key, ' ', '')))

			if aliasKey ~= '' and lookup[aliasKey] == nil then lookup[aliasKey] = entry.category end
		end
	end
	compiled.overrideLookup = lookup
	return lookup
end

local function parseAliasSetOverride(value, compiled) -- parse site override 2.0
	if type(compiled) ~= 'table' or type(compiled.entries) ~= 'table' then
		devError('PASO: expected alias set compiled by CAS, got: ' .. type(compiled))
	end

	local cleaned = trim(value)

	if normalize(cleaned) == 'none' then return {}, true end

	local lookup = overrideLookup(compiled)
	local values = {}
	local seen = {}

	for _, piece in ipairs(mw.text.split(cleaned, ',', true)) do
		local key = normalize(piece)
		local category = lookup[key]

		if key == '' or category == nil then return {}, false end

		if not seen[category] then
			seen[category] = true
			values[#values + 1] = category
		end
	end

	return values, true
end

local function parseLinkTargetOverride(value, opts)
	local namespaces = {}

	for _, namespace in ipairs((opts and opts.namespaces) or { 0 }) do
		namespaces[namespace] = true
	end

	local target = trim(value)
	local inner = string.match(target, '^%[%[(.*)%]%]$')

	if inner then
		if string.find(inner, '[%[%]]') then return {}, false end

		target = mw.text.split(inner, '|', true)[1]
	end

	target = mw.text.split(target, '#', true)[1]
	target = trim((mw.ustring.gsub(target, '^%s*:%s*', '')))

	if target == '' then return {}, false end

	local title = mw.title.new(target)
	if title and not title.isExternal and namespaces[title.namespace] then return { title.prefixedText }, true end

	return {}, false
end

local function parseExactOverride(value, map)
	if type(map) ~= 'table' then devError('PEO: expected a map of accepted values, got: ' .. type(map)) end

	local lookup = {}

	for key, mapped in pairs(map) do
		if type(key) == 'number' then key = mapped end

		if type(key) ~= 'string' or type(mapped) ~= 'string' then
			devError('PEO: map keys and values must be strings')
		end

		local normalizedKey = normalize(key)

		if lookup[normalizedKey] ~= nil and lookup[normalizedKey] ~= mapped then
			devError('PEO: "' .. key .. '" collides with another key after normalization, remove it')
		end

		lookup[normalizedKey] = mapped
	end

	local mapped = lookup[normalize(value)]
	if trim(value) ~= '' and mapped ~= nil then return { mapped }, true end

	return {}, false
end

-- // compiling complete, resolvers //
local function processValues(values, canon, filter)
	local result = {}
	local seen = {}
	local rejected = false

	for _, value in ipairs(values) do
		local processed = value

		if canon then processed = canon(processed) end

		processed = processed ~= nil and trim(processed) or ''

		if processed == '' or (filter and not filter(processed)) then
			rejected = true
		elseif not seen[processed] then
			seen[processed] = true
			result[#result + 1] = processed
		end
	end
	return result, rejected
end

local function resolve(spec)
	if type(spec) ~= 'table' then devError('resolver: expected table with named arguments, got: ' .. type(spec)) end

	local parse = spec.parse
	local canon = spec.canon
	local filter = spec.filter
	local max = spec.max

	if parse ~= nil and type(parse) ~= 'function' then
		devError('resolver: parse must be a function, got: ' .. type(parse))
	end
	if canon ~= nil and type(canon) ~= 'function' then
		devError('resolver: canon must be a function, got: ' .. type(canon))
	end
	if filter ~= nil and type(filter) ~= 'function' then
		devError('resolver: filter must be a function, got: ' .. type(filter))
	end
	if max ~= nil and (type(max) ~= 'number' or max < 1 or max % 1 ~= 0) then -- necessary check to avoid hard braps
		devError('resolver: MATH! MAX must be positive integer, got ' .. tostring(max))
	end

	local state = {
		values = {},
		source = 'detected',
		overrideProvided = trim(spec.override) ~= '',
		invalid = false,
		undetected = false,
		overflow = false,
		customized = false,
	}

	if spec.applicable == false then
		state.source = 'inapplicable'
		state.invalid = state.overrideProvided
		state.customized = state.overrideProvided or yesno(spec.suppress, false) == true
		return state
	end

	if yesno(spec.suppress, false) then
		state.source = 'suppressed'
		state.invalid = state.overrideProvided
		state.customized = true
		return state
	end

	state.customized = state.overrideProvided

	if state.overrideProvided then
		if parse == nil then devError('resolver: You provided an override, but did not provide a parse function') end

		local values, valid = parse(spec.override)
		if valid then
			if type(values) ~= 'table' then
				devError('resolver: parse must return a list of values, got ' .. type(values))
			end

			local processed, rejected = processValues(values, canon, filter)
			if not rejected and (max == nil or #processed <= max) then
				state.source = 'override'
				state.values = processed
				return state
			end
		end

		state.invalid = true
	end

	local candidates = spec.candidates or {}

	if type(candidates) ~= 'table' then devError('resolver: candidates must be a list, got ' .. type(candidates)) end

	local processed = processValues(candidates, canon, filter)
	if max and #processed > max then
		state.overflow = true
		for index = #processed, max + 1, -1 do
			processed[index] = nil
		end
	end

	state.values = processed
	state.undetected = #processed == 0
	return state
end

local function anyFlag(flag, ...)
	for index = 1, select('#', ...) do
		local state = select(index, ...)
		if type(state) == 'table' and state[flag] then return true end
	end
	return false
end

local function anyCustomized(...)
	return anyFlag('customized', ...)
end

local function anyInvalid(...)
	return anyFlag('invalid', ...)
end

local function fromMap(map, opts)
	if type(map) ~= 'table' then
		devError('fromMap: expected a table, got ' .. type(map) .. 'something seriously went wrong.')
	end

	local strict = opts ~= nil and opts.strict == true
	return function(value)
		local mapped = map[value]
		if mapped ~= nil then
			return mapped
		elseif strict then
			return nil
		end

		return value
	end
end

local errorCat = 'Pages with categorizer errors'
local missingCat = 'Pages with nonexistent categorizer categories' -- module is trying to call a redlink cat, possibly requiring admin intervention at alias data

local function makeMain(categoriesFn, namespaces)
	if type(categoriesFn) ~= 'function' then
		devError('M&Ms: expected the subject categories function, got ' .. type(categoriesFn))
	end

	local allowed = {}

	for _, namespace in ipairs(namespaces or { 0 }) do
		allowed[namespace] = true
	end

	return function(frame)
		local parent = frame:getParent()
		if not parent then return '' end
		local args = parent.args
		local title = mw.title.getCurrentTitle()

		if not allowed[title.namespace] or yesno(args.nocat, false) then return '' end

		local ok, result = pcall(categoriesFn, args)

		if not ok then -- quiet down non critical crashes, issue cat
			if isDevError(result) then error(result, 0) end
			mw.addWarning('Categorizer error: ' .. tostring(result))
			return '[[Category:' .. errorCat .. ']]'
		end

		if getmetatable(result) == List then result = result:list() end

		if type(result) ~= 'table' then
			devError('M&Ms: the categories function must return a list, but got: ' .. type(result))
		end

		local output = {}
		local missing = {}
		local failed = false

		for _, name in ipairs(result) do
			name = trim(name)

			if name ~= '' then
				local title = mw.title.new(name, categoryNamespace)

				if title then
					output[#output + 1] = '[[Category:' .. name .. ']]'
					local ok, exists = pcall(function()
						return title.exists -- expensive call, massive value
					end)
					if ok and not exists then missing[#missing + 1] = '[[:Category:' .. name .. ']]' end -- instead of silently spamming red cats, also call an administrator/maintenance editor to review & handle the new red cats
				else
					failed = true
				end
			end
		end
		if failed then output[#output + 1] = '[[Category:' .. errorCat .. ']]' end
		if #missing > 0 then
			output[#output + 1] = '[[Category:' .. missingCat .. ']]'
			mw.addWarning(
				'Categorizer added nonexistent categories. If needed, create them, otherwise check subject configs or alias lists for fix. Added: '
					.. table.concat(missing, ', ')
			)
		end

		return table.concat(output)
	end
end

p.trim = trim
p.normalize = normalize
p.devError = devError
p.need = need
p.newList = newList
p.fill = fill
p.clean = clean

p.findYears = findYears
p.findYearRanges = findYearRanges
p.findLinkTargets = findLinkTargets
p.readDataAttr = readDataAttr
p.parseNumber = parseNumber
p.pickBand = pickBand
p.compileAliasSet = compileAliasSet
p.matchAliasSets = matchAliasSets

p.parseYearOverride = parseYearOverride
p.parseAliasSetOverride = parseAliasSetOverride
p.parseLinkTargetOverride = parseLinkTargetOverride
p.parseExactOverride = parseExactOverride

p.resolve = resolve
p.anyCustomized = anyCustomized
p.anyInvalid = anyInvalid
p.fromMap = fromMap
p.makeMain = makeMain

return p