--Based on [[Module:tl-utilities]] by Ysrael214 (see module edit history for attribution)
local export = {}
local m_str_utils = require("Module:string utilities")
local m_table = require("Module:table")
local badlit_encode_module = "Module:ceb-bad_sc"
local lang = require("Module:languages").getByCode("ceb")
local sc_Tglg = require("Module:scripts").getByCode("Tglg")
local rfind = m_str_utils.find
local rmatch = m_str_utils.match
local ugmatch = m_str_utils.gmatch
local rsubn = m_str_utils.gsub
local rsplit = m_str_utils.split
local toNFC = mw.ustring.toNFC
local toNFD = mw.ustring.toNFD
local trim = mw.text.trim
local u = m_str_utils.char
local ulen = m_str_utils.len
local ulower = m_str_utils.lower
local uupper = m_str_utils.upper
local AC = u(0x0301) -- acute = ́
local GR = u(0x0300) -- grave = ̀
local CFLEX = u(0x0302) -- circumflex = ̂
local TILDE = u(0x0303) -- tilde = ̃
local DIA = u(0x0308) -- diaeresis = ̈
local MACRON = u(0x0304) -- macron = ̄
local DOTOVER = u(0x0307) -- dot over = ̇
local vowel = "aeëəiou" -- vowel
local V = "[" .. vowel .. "]"
local NV = "[^" .. vowel .. "]"
local accent = AC .. GR .. CFLEX .. MACRON
local accent_c = "[" .. accent .. "]"
local ipa_stress = "ˈˌ"
local ipa_stress_c = "[" .. ipa_stress .. "]"
local separator = accent .. ipa_stress .. "# .-"
local C = "[^" .. vowel .. separator .. "]" -- consonant
local ceb_affix_list = {
["circumfix"] = {"ka- -han", "ka- -an", "ka- -nan", "ka- -hon", "ka- -non", "ka- -on"},
["suffix"] = {"-hon", "-han", "-on", "-an", "-nan", "-non", "-ng"},
["prefix_multi"] = {
"magka-", "nagka-", "pagka-",
"maka-", "naka-",
"maga-", "naga-",
"gina-",
},
["prefix_priority"] = {"i-"},
["prefix_single"] = {
"mang-", "nang-", "pang-",
"man-", "nan-", "pan-",
"mam-", "nam-", "pam-",
"hing-", "him-", "hin-",
"tag-", "tig-",
"mag-", "nag-", "pag-",
"ma-", "na-", "pa-", "hi-", "ka-", "sa-",
"ga-", "ni-", "gi-", "mo-", "mi-",
"a-"
},
["infix"] = {"-um-", "-in-"},
}
-- Build search indices for prefixes
local prefix_multi_index = {}
local prefix_single_index = {}
for _, affix in ipairs(ceb_affix_list.prefix_multi) do
local first = affix:sub(1,1)
prefix_multi_index[first] = prefix_multi_index[first] or {}
table.insert(prefix_multi_index[first], affix)
end
for _, affix in ipairs(ceb_affix_list.prefix_single) do
local first = affix:sub(1,1)
prefix_single_index[first] = prefix_single_index[first] or {}
table.insert(prefix_single_index[first], affix)
end
--- Wrapper around rsubn() that returns only the new string.
-- @param term string Input text.
-- @param foo string Pattern to search for.
-- @param bar string|table|function Replacement, as in string.gsub.
-- @return string Text with the substitution applied once, discarding extra return values.
local function rsub(term, foo, bar)
local retval = rsubn(term, foo, bar)
return retval
end
--- Repeatedly apply rsub() until the result no longer changes.
-- @param term string Input text.
-- @param foo string Pattern to search for.
-- @param bar string|table|function Replacement, as in string.gsub.
-- @return string Stabilized text after no further substitutions are possible.
local function rsub_repeatedly(term, foo, bar)
while true do
local new_term = rsub(term, foo, bar)
if new_term == term then
return term
end
term = new_term
end
end
--- Decompose a string to NFD while preserving ñ/Ñ and ü/Ü, with optional e-dia recomposition.
-- @param text string Input text.
-- @param recompose_e_dia boolean|nil If truthy, recomposes e/E + diaeresis into ë/Ë.
-- @return string Text in canonical decomposed form, with special handling for some letters.
local function decompose(text, recompose_e_dia)
-- decompose everything but ñ and ü
text = toNFD(text)
text = rsub(text, ".[" .. TILDE .. DIA .. "]", {
["n" .. TILDE] = "ñ",
["N" .. TILDE] = "Ñ",
["u" .. DIA] = "ü",
["U" .. DIA] = "Ü",
})
if recompose_e_dia then
text = rsub(text, ".[" .. DIA .. "]", {
["e" .. DIA] = "ë",
["E" .. DIA] = "Ë",
})
end
return text
end
--- Restore capitalization based on an original caps map, respecting syllable markers.
-- @param input string Text whose capitalization needs fixing.
-- @param caps_map string Original text with correct capitalization.
-- @return string Text with capitalization restored around syllable/glottal markers.
local function fix_capitalization(input, caps_map)
local syllbreak = 0
local text = ulower(input)
local syllbreak_chars = ".7"
for i=1, #text do
local text_pre = text:sub(1, i-1)
local text_current = text:sub(i,i)
local text_post = text:sub(i+1)
local caps_current = caps_map:sub(i-syllbreak, i-syllbreak)
if rfind(text_current, "[|" .. syllbreak_chars .. "]") and not rfind(caps_current, "[" .. syllbreak_chars .. "]")then
syllbreak = syllbreak + 1
elseif uupper(text_current) == caps_current then
text = table.concat({text_pre, uupper(text_current), text_post})
end
end
return text
end
--- Remove diacritic accent marks from a string (acute, grave, circumflex, macron).
-- @param str string Text potentially containing combining accent marks.
-- @return string Text with accents stripped but base letters preserved.
function export.remove_accents(str)
str = decompose(str, "recompose e-dia")
str = rsub(str, "(.)" .. accent_c, "%1")
return str
end
--- Decode badlit text into a Latin respelling
-- @param text string Input possibly containing badlit characters.
-- @return string Text where badlit segments are transliterated to Latin.
function export.decode_badlit(text)
local text = rsub(text, "[ᜀ-ᜟ᜵᜶]+", function(badlit)
result = lang:transliterate(badlit, sc_Tglg)
result = rsub(result, "([aeiou])([aeiou])", "%1-%2")
result = rsub(result, "%-", "7")
result = rsub(result, "([aeiou])", "%1" .. MACRON) -- No way to know stress in badlit. Disable for now.
return result
end)
return text
end
-- "Align" syllabified respelling `syllab` to original spelling `spelling` by matching character-by-character, allowing
-- for extra syllable and accent markers in the syllabification and certain mismatches in the consonants. The goal is to
-- produce the appropriately syllabified version of the original spelling (the pagename) by matching characters in the
-- syllabified respelling to the original spelling, putting the syllable boundaries in the appropriate places in the
-- original spelling. As an example, given syllabified respelling 'a.ma.7ín' and original spelling 'amain', we would
-- like to produce 'a.ma.in'.
--
-- If we encounter an extra syllable marker (.), we allow and keep it. If we encounter an extra accent marker in thes
-- syllabification, we drop it. We allow for mismatches in capitalization and for certain other mismatches, e.g. extra
-- glottal stops (written 7), h in respelling vs. g or j in the original, etc. If we can't match, we return nil
-- indicating the alignment failed.
-- @param syllab string Syllabified respelling (with . and 7, etc.).
-- @param spelling string Original orthographic form (e.g., page name).
-- @return string|nil Syllabified original spelling, or nil if alignment fails.
function export.align_syllabification_to_spelling(syllab, spelling)
local result = {}
local function concat_result()
-- Postprocess to remove dots (syllable boundaries) next to hyphens.
return (toNFC(table.concat(result)):gsub("%.%-", "-"):gsub("%-%.", "-"))
end
-- Remove glottal stop (7) from respelling to simplify the code below, because it's never found in the original
-- spelling. (FIXME: We should do the same for diacritics, but they're currently removed earlier, in
-- syllabify_from_spelling(). We should probably get rid of the removal there and put it here.)
syllab = decompose(syllab:gsub("ː", ""), "recompose e-dia"):gsub("7", "")
spelling = decompose(spelling, "recompose e-dia")
local syll_chars = rsplit(ulower(syllab), "")
local spelling_chars = rsplit(spelling, "")
local i = 1
local j = 1
local function matches(uci, ucj)
-- Return true if a syllabified respelling character (uci) matches the corresponding spelling char (ucj).
-- Both uci and ucj should be lowercase.
-- Sound is at the key, values are the letters sound can match
local matching_chars = {
["b"] = {"v"},
["e"] = {"i"},
["ë"] = {"a", "e", "o", "u"},
["h"] = {"g", "j", "x"},
["i"] = {"e", "y"},
["j"] = {"g"},
["k"] = {"c", "j"},
["o"] = {"u"},
["p"] = {"f"},
["s"] = {"j", "c", "x", "z"},
["u"] = {"o"},
["w"] = {"u", "o"},
["y"] = {"i"}
}
return uci == ucj or (matching_chars[uci] and m_table.contains(matching_chars[uci], ucj) and true) or false
end
local function silent_spelling_letter(ucj)
return ucj == "h" or ucj == "'" or ucj == "-"
end
local function syll_at(pos)
return syll_chars[pos] or ""
end
local function spell_at(pos)
return spelling_chars[pos] or ""
end
local function uspell_at(pos)
local c = spelling_chars[pos]
return c and ulower(c) or ""
end
while i <= #syll_chars or j <= #spelling_chars do
local uci = syll_at(i)
local cj = spell_at(j)
local ucj = uspell_at(j)
if uci == "g" and syll_at(i - 1) == "n" and syll_at(i + 1) == "." and matches(syll_at(i + 2), ucj) and
not matches(syll_at(i + 2), uspell_at(j + 1)) then
-- As a special case, before checking whether the corresponding characters match, we have to skip an extra
-- g in an -ng- sequence in the syllabified respelling if the corresponding spelling character matches the
-- next respelling character (taking into account the syllable boundary). This is so that e.g.
-- syll='ba.rang.gay' matches spelling='barangay'. Otherwise we will match the first respelling g against
-- the spelling g and the second respelling g won't match. A similar case occurs with
-- syll='E.vang.he.lis.ta' and spelling='Evangelista'. But we need an extra condition to not do this hack
-- when syll='ba.rang.gay' matches spelling='baranggay'.
i = i + 1
elseif uci == "g" and ucj == "g" and uspell_at(j + 1) == TILDE then
table.insert(result, cj)
table.insert(result, uspell_at(j + 1))
i = i + 1
j = j + 2
elseif uci == "f" and ucj == "p" and uspell_at(j + 1) == "h" then
table.insert(result, cj)
table.insert(result, uspell_at(j + 1))
i = i + 1
j = j + 2
elseif matches(uci, ucj) then
table.insert(result, cj)
i = i + 1
j = j + 1
elseif ucj == uspell_at(j - 1) and uci == "." and ucj ~= syll_at(i + 1) then
-- See below. We want to allow for a doubled letter in spelling that is pronounced single, and preserve the
-- doubled letter. But it's tricky in the presence of syllable boundaries on both sides of the doubled
-- letter as well as doubled letters pronounced double. Specifically, there are three possibilities,
-- exemplified by:
-- (1) syll='Mal.lig', spelling='Mallig' -> 'Mal.lig';
-- (2) syll='Ma.lig', spelling='Mallig' -> 'Ma.llig';
-- (3) syll='Wil.iam', spelling='William' -> 'Will.iam'.
-- If we copy the dot first, we get (1) and (2) right but not (3).
-- If we copy the double letter first, we get (2) and (3) right but not (1).
-- We choose to copy the dot first except in the situation exemplified by (3), where we copy the doubled
-- letter first. The condition above handles (3) (the doubled letter matches against a dot) while not
-- interfering with (1) (where the doubled letter also matches against a dot but the next letter in the
-- syllabification is the same as the doubled letter, because the doubled letter is pronounced double).
table.insert(result, cj)
j = j + 1
elseif silent_spelling_letter(ucj) and uci == "." and ucj ~= syll_at(i + 1) and
not rfind(uspell_at(j + 1), V) then
-- See below for silent h or apostrophe in spelling. This condition is parallel to the one directly above
-- for silent doubled letters in spelling and handles the case of syllab='Abduramán', spelling='Abdurahman',
-- which should be syllabified 'Ab.du.rah.man'. But we need a check to see that the next spelling character
-- isn't a vowel, because in that case we want the silent letter to go after the period, e.g.
-- syllab='Jumu7á', spelling='Jumu'ah' -> 'Ju.mu.'ah' (the 7 is removed above).
table.insert(result, cj)
j = j + 1
elseif uci == "." then
table.insert(result, uci)
i = i + 1
elseif ucj == uspell_at(j - 1) then
-- A doubled letter in spelling that is pronounced single. Examples:
-- * syllab='Ma.líg', spelling='Mallig' -> 'Ma.llig' (with l)
-- * syllab='Lu.il.yér', spelling='Lhuillier' -> 'Lhu.ill.ier' (with l; a more complex example)
-- * syllab='a.sa.la.mu a.lai.kum', spelling='assalamu alaikum' -> 'as.sa.la.mu a.lai.kum' (with s)
-- * syllab='Jé.fer.son', spelling='Jefferson' -> 'Je.ffer.son' (with f)
-- * syllab='Je.ma', spelling='Gemma' -> 'Ge.mma' (with m)
-- * syllab='Ha.na', spelling='Hannah' -> 'Ha.nnah' (with n)
-- * syllab='A.by', spelling='Abby' -> 'A.bby' (with b)
-- * syllab='Ka.ba', spelling='Kaaba' -> 'Kaa.ba' (with a)
-- * syllab='Fu.ji', spelling='Fujii' -> 'Fu.jii' (with i)
table.insert(result, cj)
j = j + 1
elseif silent_spelling_letter(ucj) and not (ucj == "h" and rfind(uspell_at(j - 1), V) and rfind(uspell_at(j + 1), V)) then
-- A silent h, apostrophe or hyphen in spelling. Examples:
-- * syllab='adán', spelling='adhan' -> 'a.dhan'
-- * syllab='Atanasya', spelling='Athanasia' -> 'A.tha.nas.ia'
-- * syllab='Cýntiya', spelling='Cynthia' -> 'Cyn.thi.a'
-- * syllab='Ermóhenes', spelling='Hermogenes' -> 'Her.mo.ge.nes'
-- * syllab='Abduramán', spelling='Abdurahman' -> 'Ab.du.rah.man'
-- * syllab='Jumu7á', spelling='Jumu'ah' -> 'Ju.mu.'ah'
-- * syllab='pag7andam', spelling='pag-andam' -> 'pag-an.dam'
table.insert(result, cj)
j = j + 1
elseif uci == AC or uci == GR or uci == CFLEX or uci == DIA or uci == TILDE or uci == MACRON or
uci == "y" or uci == "w" then
-- skip character
i = i + 1
else
-- non-matching character
mw.log(("Syllabification alignment mismatch for pagename '%s' (position %s, character %s), syllabified respelling '%s' (position %s, character %s), aligned result so far '%s'"
):format(spelling, j, ucj, syllab, i, uci, concat_result()))
return nil
end
end
if i <= #syll_chars or j <= #spelling_chars then
-- left-over characters on one side or the other
mw.log(("Syllabification alignment mismatch for pagename '%s' (%s), syllabified respelling '%s' (%s), aligned result so far '%s'"
):format(
spelling, j > #spelling_chars and "end of string" or ("position %s, character %s"):format(j, uspell_at(j)),
syllab, i > #syll_chars and "end of string" or ("position %s, character %s"):format(i, syll_at(i)),
concat_result()))
return nil
end
return concat_result()
end
--- Check if a string contains any badlit characters.
-- @param text string Input text.
-- @return string|nil First matching badlit character, or nil if none (truthy/falsey test).
function export.has_badlit(text)
return text:match("[ᜀ-ᜟ]")
end
--- Canonicalize whitespace: collapse multiple spaces and trim leading/trailing spaces.
-- @param text string Input text.
-- @return string Text with single spaces and no leading/trailing space.
local function canon_spaces(text)
text = rsub(text, "%s+", " ")
text = rsub(text, "^ ", "")
text = rsub(text, " $", "")
return text
end
--- Automatically syllabify a spelling (and badlit segments) according to Cebuano rules.
-- @param text string Input spelling or respelling.
-- @param pagename string Page name used for some heuristics (e.g. hyphen alignment).
-- @return string Syllabified text using periods as syllable separators.
function export.syllabify_from_spelling(text, pagename)
-- Auto syllabifications start --
local vowel = vowel .. "ẃý" -- vowel
local V = "[" .. vowel .. "]"
local NV = "[^" .. vowel .. "]"
local C = "[^" .. vowel .. separator .."]" -- consonant
text = trim(text)
text = canon_spaces(text)
text = rsub(text, "[ᜀ-ᜟ]+", function(badlit)
return "<᜶" .. export.decode_badlit(badlit) .. "᜶>"
end)
text = decompose(text, "recompose e-dia")
local origtext = text
text = string.lower(text)
text = rsub(text, "[.] ", "․ ")
text = rsub(text, "[.]$", "․")
-- put # at word beginning and end and double ## at text/foot boundary beginning/end
text = rsub(text, " | ", "# | #")
text = "##" .. rsub(text, " ", "# #") .. "##"
text = rsub_repeatedly(text, "([.]?)#([.]?)", "#")
-- special word "mga"
text = rsub(text, "#mga#", "#m.ga#")
text = rsub(text, "ng̃", "ŋ")
text = rsub(text, "ng", "ŋ")
text = rsub(text, "g̃", "ġ")
text = rsub(text, "ch", "ĉ")
text = rsub(text, "t_s", "ć")
text = rsub(text, "sh", "ʃ")
text = rsub(text, "gu([eëiy])", "ǵ%1")
text = rsub(text, "qu([eëiy])", "ḱ%1")
text = rsub(text, "r", "ɾ")
text = rsub(text, "ɾɾ", "r")
text = rsub(text, "ʔ", "7")
-- double dot improvements
text = rsub(text, "([dinstz])%.y", "%1..y")
text = rsub(text, "n%.k", "n..k")
text = rsub_repeatedly(text, "#(" .. C .. "+)i([aeou])","#%1i.%2")
text = rsub_repeatedly(text, "#(" .. C .. "+)u([aeio])","#%1u.%2")
text = rsub_repeatedly(text, "(" .. C .. ")(%1)i([aeou])","%1%2.i%3")
text = rsub_repeatedly(text, "(" .. C .. ")(%1)u([aeio])","%1%2.u%3")
text = rsub_repeatedly(text, "(" .. C .. ")(" .. C .. ")i([aeou])","%1%2i.%3")
text = rsub_repeatedly(text, "(" .. C .. ")(" .. C .. ")u([aeio])","%1%2u.%3")
text = rsub_repeatedly(text, "(" .. V .. accent_c .. "*)(" .. C .. ")u([aeio])","%1%2.u%3")
text = rsub_repeatedly(text, "(" .. V .. accent_c .. "*)(" .. C .. ")i([aeou])","%1%2.i%3")
text = rsub_repeatedly(text, "(" .. V .. accent_c .. "*)u([aeio])","%1.u%2")
text = rsub_repeatedly(text, "(" .. V .. accent_c .. "*)o([aei])","%1.ó%2")
text = rsub(text, "a(" .. accent_c .. "*)o([#.7])","a%1ó%2")
-- eu rules
text = rsub_repeatedly(text, "([^" .. vowel .. "#])([e])(" .. accent_c .. "?)([u])(" .. accent_c .. "?)","%1%2%3.%4%5")
text = rsub(text, "y([ˈˌ." .. accent .. "]*)([bćĉdfgǵhjĵkḱlmnɲŋpɾrsʃtvwyɟzʔ#" .. vowel .. "])","ý%1%2")
text = rsub(text, "ý([ˈˌ." .. accent .. "]*)([bćĉdfgǵhjĵkḱlmnɲŋpɾrsʃtvwyɟzʔ#])([wyẃý])","y%1%2%3")
text = rsub(text, "ý(" .. V .. ")", "y%1")
text = rsub(text, "w([ˈˌ]?)([bćĉdfgǵjĵkḱlmnɲŋpɾrsʃtvwyɟzʔ#" .. vowel .. "])","ẃ%1%2")
text = rsub(text, "ẃ([ˈˌ." .. accent .. "]*)([bćĉdfgǵhjĵkḱlmnɲŋpɾrsʃtvwyɟzʔ#])([wyẃý])","w%1%2%3")
text = rsub(text, "ẃ(" .. V .. ")","w%1")
text = rsub(text, "(" .. V .. ")(" .. accent_c .. "?)ẃ([bdfgǵkḱpt])([ɾr" .. vowel .. separator .."])" ,"%1%2w%3%4")
text = rsub(text, "(" .. V .. ")(" .. accent_c .. "?)ẃ([bfgǵkḱp])([l" .. vowel .. separator .."])" ,"%1%2w%3%4")
text = rsub(text, "(" .. V .. ")(" .. accent_c .. "?)ý([bdfgǵkḱpt])([ɾr" .. vowel .. separator .."])" ,"%1%2y%3%4")
text = rsub(text, "(" .. V .. ")(" .. accent_c .. "?)ý([bfgǵkḱp])([l" .. vowel .. separator .."])" ,"%1%2y%3%4")
text = rsub_repeatedly(text, "(" .. V .. accent_c .. "*)(" .. C .. V .. ")", "%1.%2")
-- "mb", "mp", "nd", "nk", "nt" combinations
text = rsub_repeatedly(text, "(m)([bp])([^lɾrɟyw" .. vowel .. separator .."])(" .. V .. ")", "%1%2.%3%4")
text = rsub_repeatedly(text, "(n)([dk])([^lɾrɟyw" .. vowel .. separator .. "])(" .. V .. ")", "%1%2.%3%4")
text = rsub_repeatedly(text, "(n)([s])([^ɟyw" .. vowel .. separator .. "])(" .. V .. ")", "%1%2.%3%4")
text = rsub_repeatedly(text, "(n)([t])([^lɾrɟyws" .. vowel .. separator .. "])(" .. V .. ")", "%1%2.%3%4")
text = rsub_repeatedly(text, "(ŋ)([k])([^lɾrɟwy" .. vowel .. separator .. "])(" .. V .. ")", "%1%2.%3%4")
text = rsub_repeatedly(text, "([lɾr])([bćĉdfgǵkḱlmnpʃvz])([^lɾrɟwy" .. vowel .. separator .. "])(" .. V .. ")", "%1%2.%3%4")
text = rsub_repeatedly(text, "([lɾr])([s])([^lɾrɟyw" .. vowel .. separator .. "])(" .. V .. ")", "%1%2.%3%4")
text = rsub_repeatedly(text, "([lɾr])([t])([sz]?)([^lɾrɟysʃw" .. vowel .. separator .. "])(" .. V .. ")", "%1%2%3.%4%5")
text = rsub_repeatedly(text, "(s)([ktp])([^lɾrwɟy" .. vowel .. separator .. "])(" .. V .. ")", "%1%2.%3%4")
text = rsub_repeatedly(text, "(" .. V .. accent_c .. "*" .. C .. ")(" .. C .. V .. ")", "%1.%2")
text = rsub_repeatedly(text, "(" .. V .. accent_c .. "*" .. C .. "+)(" .. C .. C .. V .. ")", "%1.%2")
text = rsub_repeatedly(text, "(" .. C .. ")%.s([^ɟwy" .. vowel .. separator .. "])", "%1s.%2")
-- Any aeëo, or stressed iu, should be syllabically divided from a following aeëo or stressed iu.
text = rsub_repeatedly(text, "([aeëo]" .. accent_c .. "*)([aeëo])", "%1.%2")
text = rsub_repeatedly(text, "([aeëo]" .. accent_c .. "*)(" .. V .. accent_c .. ")", "%1.%2")
text = rsub(text, "([iu]" .. accent_c .. ")([aeëo])", "%1.%2")
text = rsub_repeatedly(text, "([iu]" .. accent_c .. ")(" .. V .. accent_c .. ")", "%1.%2")
text = rsub_repeatedly(text, "i(" .. accent_c .. "*)i", "i%1.i")
text = rsub_repeatedly(text, "u(" .. accent_c .. "*)u", "u%1.u")
text = rsub(text, "ĉ", "ch")
text = rsub(text, "ć", "ts")
text = rsub(text, "ŋ", "ng")
text = rsub(text, "ʃ", "sh")
text = rsub(text, "ǵ.([ei])", "g.u%1")
text = rsub(text, "ǵ", "gu")
text = rsub(text, "ġ", "g̃")
text = rsub(text, "ḱ.([ei])", "q.u%1")
text = rsub(text, "ḱ", "qu")
text = rsub(text, "r", "rr")
text = rsub(text, "ɾ", "r")
text = rsub_repeatedly(text, "([.]+)", ".")
text = rsub(text, "[.]?-[.]?", "-")
text = rsub(text, "[‿]([^ ])", "|%1")
text = rsub(text, "[.]([^ ])", "|%1")
text = rsub(text, "([|])+", "%1")
-- remove # symbols at word and text boundaries
text = rsub_repeatedly(text, "([.]?)#([.]?)", "")
text = rsub(text, "․", ".")
text = rsub(text, "ẃ", "w")
text = rsub(text, "ý", "y")
-- Fix Capitalization --
text = fix_capitalization(text, origtext)
-- Fix hyphens --
-- FIXME!!! Why are we relying on looking at the pagename here? This should not be happening.
origtext = pagename
if (table.concat(rsplit(origtext, "-")) == table.concat(rsplit(table.concat(rsplit(text, "|")), "-"))) then
syllbreak = 0
for i=1, #text do
if text:sub(i,i) == "|" then
if origtext:sub(i-syllbreak, i-syllbreak) == "-" then
text = table.concat({text:sub(1, i-1), "-", text:sub(i+1)})
else
syllbreak = syllbreak + 1
end
end
end
end
-- Reencode badlit
text = rsub(text, "[<][᜶]([^᜶]+)[᜶][>]", function(badlit)
badlit = badlit:gsub("|", "/"):gsub("7", "")
local result = require(badlit_encode_module).transcribe(badlit:gsub("|", "/"), false, false, false)
result = rsub(result, " ᜵ ", "|")
return result
end)
-- FIXME! Hack -- up above we changed periods to vertical bars. The rest of the code expects periods so change
-- them back. We should clean up the code above to leave the periods alone.
return (text:gsub("|", "%."))
end
--- Convenience wrapper to syllabify a respelling and align it to a pagename.
-- @param respelling string Syllabification-friendly respelling.
-- @param pagename string|nil Page name; defaults to respelling if nil.
-- @return string|nil Syllabified and aligned pagename, or nil if alignment fails.
function export.syllabify_and_align(respelling, pagename)
if pagename == nil then
pagename = respelling
end
local syllabification = export.syllabify_from_spelling(respelling, pagename)
return export.align_syllabification_to_spelling(syllabification, pagename)
end
--- Apply Cebuano nasal assimilation rules to a prefix + stem combination.
-- @param text1 string Left part (typically a nasal-ending prefix).
-- @param text2 string Right part (typically the root/stem).
-- @param assimilation string|nil "none", "partial", or "total" (default is behaves like "none").
-- @return string Combined string with nasal assimilation applied.
local function nasal_adjust(text1, text2, assimilation)
local t1 = text1
local t2 = text2
if assimilation ~= nil and not m_table.contains({"none", "partial", "total"}, assimilation) then
error('Assimilation options can only be "none", "partial", or "total".')
end
t2 = rsub(t2, "^ng", "ŋ")
t1 = rsub(t1, "ng([- ]*)$", "ŋ%1")
t1 = rsub(t1, "m([- ]*)$", "ṃ%1")
t1 = rsub(t1, "n([- ]*)$", "ṇ%1")
local result = t1 .. t2
if assimilation == "partial" then
result = rsub(result, "[ŋṇ]([- ]*)([bp])", "m%1%2")
result = rsub(result, "[ŋ]([- ]*)([dlnst])", "n%1%2")
result = rsub(result, "[ṇ]([- ]*)([kgʔ])", "ŋ%1%2")
elseif assimilation == "total" then
result = rsub(result, "[ŋṇṃ][- ]*([bp])(" .. V .. ")%1([lr]?)%2(" .. NV .. "+)(" .. V .. ")", "m%2m%3%2%4%5")
result = rsub(result, "[ŋṇṃ][- ]*([bp])([lr]?)(" .. V .. ")%1%2%3(" .. NV .. "+)(" .. V .. ")", "m%2%3m%2%3%4%5")
result = rsub(result, "[ŋṇ][- ]*([dnst])(" .. V .. ")%1([lr]?)%2(" .. NV .. "+)(" .. V .. ")", "n%2n%3%2%4%5")
result = rsub(result, "[ŋṇ][- ]*([dnst])([lr]?)(" .. V .. ")%1%2%3(" .. NV .. "+)(" .. V .. ")", "n%2%3n%2%3%4%5")
result = rsub(result, "[ŋṇ][- ]*([d])(" .. V .. ")r%2(" .. NV .. "+)(" .. V .. ")", "n%2n%2%3%4")
result = rsub(result, "[ŋṇ][- ]*([kgʔ])(" .. V .. ")%1([lr]?)%2(" .. NV .. "+)(" .. V .. ")", "ŋ%2ŋ%3%2%4%5")
result = rsub(result, "[ŋṇ][- ]*([kgʔ])([lr]?)(" .. V .. ")%1%2%3(" .. NV .. "+)(" .. V .. ")", "ŋ%2%3ŋ%2%3%4%5")
result = rsub(result, "[ŋṇṃ][- ]*([bp])", "m")
result = rsub(result, "[ŋṇ][- ]*([dnst])", "n")
result = rsub(result, "[ŋṇ][- ]*([kgʔ])", "ŋ")
result = rsub(result, "[ŋṇ]([- ]*)([l])", "n%1%2")
end
result = rsub(result, "ŋ", "ng")
result = rsub(result, "ṃ", "m")
result = rsub(result, "ṇ", "n")
return result
end
--- Optionally change an initial d in a segment to r depending on context and options.
-- Modify the d-r option in add affix
-- options = {
-- ["d-r"] = Values can be no value, "d", "r"
-- }
-- No value (default): If "d" does not follow any of the following consonants "d", "l", "r", then "d" would change to "r" if between vowels
-- "between": "d" would change to "r" if between vowels (regardless of the next consonant)
-- "d": Retain "d" as "d"
-- "r": Force "d" as "r"
-- @param text1 string Left context (usually material before the segment).
-- @param text2 string Segment that may start with d.
-- @param d string|nil "d", "r", "between", or nil for default behavior.
-- @return string text2 possibly modified with initial d → r.
local function d_r_adjust_root(text1, text2, d)
local t1 = text1
local t2 = text2
local t2_start_d = rmatch(t2, "^d")
if not t2_start_d or d == "d" then
return t2
end
local d_valid_consonant_check = string.sub(rsub(t2, V, ""), 2, 2) -- Get consonant after "d"
d_valid_consonant_check = not rmatch(d_valid_consonant_check, "[dlr]")
if d_valid_consonant_check or d == "between" or d == "r" then
local t1_end_vowel = rmatch(t1, V .. "$")
local t2_d_before_vowel = rmatch(t2, "^d" .. V)
if (d == "r") or (t1_end_vowel and t2_d_before_vowel) then
t2 = rsub(t2, "^d", "r")
end
end
return t2
end
--- Compute a syncoped form of a root according to metathesis and syncope rules.
-- @param root string Original root candidate.
-- @param options table Options table (e.g. options.metathesis boolean).
-- @return string Syncoped (or original) root, depending on pattern.
local function get_syncoped_root(root, options)
local result = root
local v1, c1, v2, c2 = rmatch(result, "(" .. V .. ")(" .. C .. ")(" .. V .. ")(" .. C .. "?)$")
if c2 == "" or c2 == nil then
c2 = "ʔ"
end
if not v1 then
return root
end
local syncope_stem = ""
if c1 == "h" then
if v1 ~= "a" then
syncope_stem = v1 .. c2
else
syncope_stem = v2 .. c2
end
else
syncope_stem = v1 .. c1 .. c2
if options["metathesis"] then
-- By default, the letter order is as follows (not sure here, figuring out sonority and articulation hierarchies)
local ceb_metathesis_order = {"mpbfv", "sz", "tnŋkg", "yw", "dr", "l", "h", "ʔ"}
-- build rank lookup
local letter_rank = {}
for i, group in ipairs(ceb_metathesis_order) do
for c in ugmatch(group, ".") do
letter_rank[c] = i
end
end
local meta_switch = options["metathesis"] == 1 or options["metathesis"] == true
meta_switch = meta_switch and letter_rank[c1] > letter_rank[c2]
if meta_switch and c1 == "l" and m_table.contains({"s", "z", "t", "k", "g"}, c2) then
meta_switch = false
end
-- Don't do metathesis if has semivowel
if meta_switch and (m_table.contains({"w", "y"}, c1) or m_table.contains({"w", "y"}, c2)) then
meta_switch = false
end
meta_switch = meta_switch or options["metathesis"] == 2
if meta_switch then
syncope_stem = v1 .. c2 .. c1
end
end
-- Assimilation corrections
syncope_stem = rsub(syncope_stem, "r", "d")
syncope_stem = rsub(syncope_stem, "([dst])ŋ", "%1n")
syncope_stem = rsub(syncope_stem, "[nŋ]([bp])", "m%1")
syncope_stem = rsub(syncope_stem, "[ŋ]([dst])", "n%1")
syncope_stem = rsub(syncope_stem, "[n]([gk])", "ŋ%1")
end
syncope_stem = rsub(syncope_stem, "ʔ", "")
return rsub(result, "(" .. V .. ")(" .. C .. ")(" .. V .. ")(" .. C .. "?)$", syncope_stem)
end
--- Add a single prefix to a root, with nasal assimilation and other adjustments.
-- @param root string Root or word to be prefixed.
-- @param affix string Prefix without surrounding spaces (e.g. "mag-").
-- @param options table Options as in export.add_affix (assimilation, phonetic, etc.).
-- @return string Root with prefix applied (and possibly assimilated).
local function add_prefix(root, affix, options)
local hyphen = ''
local result = ulower(root)
local root_vowel_start = rfind(result, "^(" .. V .. ")")
local affix_consonant_end = rmatch(affix, C .. "$")
if root_vowel_start then
result = "ʔ" .. result
result = rsub(result, "^ʔ(.)%1", "ʔ%1ʔ%1")
end
if options["add_hyphen"] or (root_vowel_start and affix_consonant_end) or (ulower(root) ~= root) or (not rmatch(root, "^[a-z]")) then
hyphen = '-'
end
affix = affix .. hyphen
result = d_r_adjust_root(affix, result, options["d-r"])
result = nasal_adjust(affix, result, options["assimilation"])
if(ulower(root) ~= root) then
result = rsub(result, ulower(root) .. "$", root)
end
result = rsub(result, "[-]+", "-")
result = rsub(result, "ʔ", "")
return result
end
--- Add a single infix to a root according to Cebuano infixation patterns.
-- @param root string Root or word to be infixed.
-- @param affix string Infix without dashes (e.g. "um", "in").
-- @param options table Options as in export.add_affix (keep_cluster, phonetic, etc.).
-- @return string Root with infix applied (and optionally syllabified).
local function add_infix(root, affix, options)
local result = root
local new_affix = affix
local first_vowel_idx = rfind(result, V)
local special_start_conditions = {V, "ng", "ts", "ch", "sh"}
local has_special_start = false
for idx, starting in ipairs(special_start_conditions) do
has_special_start = has_special_start or rfind(result, "^" .. starting)
if has_special_start then break end
end
if options["add_hyphen"] then
if first_vowel_idx ~= 1 then
new_affix = "-" .. new_affix
end
new_affix = new_affix .. "-"
end
if options["keep_cluster"] or has_special_start then
result = string.sub(result, 1, first_vowel_idx-1) .. new_affix .. string.sub(result, first_vowel_idx)
else
result = string.sub(result, 1, 1) .. new_affix .. string.sub(result, 2)
end
result = rsub(result, "ŋ", "ng")
return result
end
--- Add a single suffix to a root, handling syncope, glottal, and vowel alternations.
-- @param root string Root or word to be suffixed.
-- @param affix string Suffix without surrounding spaces (e.g. "-an", "-in", "-hin" stripped to "an", "in", "hin").
-- @param options table Options as in export.add_affix (glottal, e-i, o-u, syncope, metathesis, etc.).
-- @return string Root with suffix applied (and optionally syllabified).
local function add_suffix(root, affix, options)
local result = root
local new_affix = affix
local ceb_native_verb_suffixes = {
"an", "on", "i", "a"
}
local glottal = options["glottal"]
-- Allow diacritics on input
result = decompose(result, true)
if rmatch(result, V .. "[" .. GR .. CFLEX .."]$") then
glottal = not (glottal == false)
end
result = rsub(result, "[" .. AC .. GR .. CFLEX .."]$", "")
result = toNFC(result)
if options["syncope"] and not options["add_hyphen"] and m_table.contains({"han", "hon", "hi", "ha", "nan", "non", "ni", "na"}, affix) then
glottal = false
local suf_c, suf_nat = rmatch(affix, "^(" .. C .. ")(.*)$")
result = result .. suf_c
new_affix = suf_nat
end
result = rsub(result, "ng", "ŋ")
if new_affix == "ng" then
result = rsub(result, "(" .. V .. ")n$", "%1")
elseif m_table.contains(ceb_native_verb_suffixes, new_affix) then
if rmatch(result, V .. "$") and not glottal then
if options["syncope"] and not options["add_hyphen"] then
result = result .. "h"
else
new_affix = "h" .. new_affix
end
end
if rmatch(result, V .. "d$") and
rmatch(new_affix, "^" .. V) and
options["d-r"] ~= "d" and
not options["add_hyphen"] then
result = rsub(result, "(" .. V .. ")" .. "d$", "%1r")
end
result = "#" .. result
if options["e-i"] == nil then
result = rsub(result, "([^e#])e(" .. C .. "?)$", "%1i%2")
elseif options["e-i"] == "i" then
result = rsub(result, "([^#])ee(" .. C .. "?)$", "%1ii%2")
result = rsub(result, "([^#])e(" .. C .. "?)$", "%1i%2")
end
if options["o-u"] == nil then
result = rsub(result, "([^ou#])o(" .. C .. "?)$", "%1u%2")
elseif options["o-u"] == "u" then
result = rsub(result, "([^#])oo(" .. C .. "?)$", "%1uu%2")
result = rsub(result, "([^#])o(" .. C .. "?)$", "%1u%2")
end
result = rsub(result, "#", "")
end
if options["add_hyphen"] then
new_affix = "-" .. new_affix
end
if options["syncope"] then
result = get_syncoped_root(result, options)
end
result = rsub(result, "ŋ", "ng")
result = result .. new_affix
return result
end
--- Guess affix type (prefix, suffix, infix, circumfix) based on dash placement.
-- @param affix string Affix spelling including '-' where applicable.
-- @return string One of "prefix", "suffix", "infix", "circumfix", or "" if unknown.
local function get_affix_type(affix)
assert(type(affix) == "string", "Expected string for affix")
affix = canon_spaces(affix)
local affix_type = ""
local has_beginning_hyphen = rfind(affix, "^-")
local has_ending_hyphen = rfind(affix, "-$")
if has_beginning_hyphen and has_ending_hyphen then
affix_type = "infix"
elseif has_beginning_hyphen then
affix_type = "suffix"
elseif has_ending_hyphen then
affix_type = "prefix"
end
return affix_type
end
--- Add one or more affixes (prefix/infix/suffix) to a root or phrase.
-- TODO
-- Diacritics (optional)
-- Badlit?
-- @param root string Base word or multi-word string.
-- @param affix string Affix or space-separated sequence of affixes.
-- @param options table|nil Affix options:
-- options = {
-- ["assimilation"] = See nasal_adjust()
-- ["wordct"] = Word to be affixed
-- ["add_hyphen"] = Force hyphen boolean
-- ["split_hyphen"] = Hyphenated words should be considered as one unit if false
-- ["keep_cluster"] = Keep consonant cluster infix
-- ["glottal"] = When adding suffix, add to prevent adding "h" at end of root ending with vowel
-- ["syllabify"] = Return syllabification data
-- ["d-r"] = See d_r_adjust_root()
-- ["e-i"] = When adding suffix, add to force ending "e" to become "i"
-- ["o-u"] = When adding suffix, add to force ending "o" to become "u"
-- ["phonetic"] = Replace appended strings with phonetic spelling
-- ["syncope"] = Attempt to do syncopation if applicable
-- ["metathesis"] = Metathesis if syncope
-- }
-- @return string Word with affix(es) applied; optionally syllabified if options.syllabify is true.
function export.add_affix(root, affix, options)
assert(type(root) == "string", "Expected string for root")
assert(type(affix) == "string", "Expected string for affix")
if options == nil then
options = {}
elseif type(options) ~= "table" then
error("Options parameter must be a table or nil.")
end
local affix_actions = {
["prefix"] = add_prefix,
["infix"] = add_infix,
["suffix"] = add_suffix
}
local words = rsplit(canon_spaces(root), " ")
local word_idx = 1
if(options["wordct"] == 0) then
words = {canon_spaces(root)}
options["split_hyphen"] = true
end
if tonumber(options["wordct"]) ~= nil and tonumber(options["wordct"]) >= 1 then
options["wordct"] = tonumber(options["wordct"])
else
options["wordct"] = 1
end
for i=1, #words do
local hyph_words = rsplit(words[i], "-")
if options["split_hyphen"] == false then
hyph_words = {words[i]}
end
for j=1, #hyph_words do
if (word_idx == options["wordct"]) then
local affixes = rsplit(canon_spaces(affix), " ")
for i=1, #affixes do
local new_affix = affixes[i]
local affix_type = get_affix_type(new_affix)
new_affix = rsub(new_affix, "^-", "")
new_affix = rsub(new_affix, "-$", "")
hyph_words[j] = affix_actions[affix_type](hyph_words[j], new_affix, options)
end
end
hyph_words[j] = rsub(hyph_words[j], "ʔ", "")
word_idx = word_idx + 1
end
words[i] = table.concat(hyph_words, '-')
end
words = table.concat(words, " ")
words = export.remove_accents(words)
if options["syllabify"] then
return export.syllabify_and_align(words)
end
return words
end
--- Reduplicate a root by syllables or fully, with various phonological options.
-- @param root string Base word to reduplicate.
-- @param syllcount number|string|nil Number of syllables to reduplicate or "all".
-- @param options table|nil Options:
-- wordct, phonetic, add_hyphen, keep_cluster, d-r, etc.
-- @return string Word with reduplication applied.
function export.reduplicate(root, syllcount, options)
local syllable_count = syllcount
if options == nil then
options = {}
elseif type(options) ~= "table" then
error("Options parameter must be a table or nil.")
end
local word_idx = 1
if tonumber(options["wordct"]) ~= nil and tonumber(options["wordct"]) >= 1 then
options["wordct"] = tonumber(options["wordct"])
else
options["wordct"] = 1
end
if syllable_count == nil then
syllable_count = 1
end
local words = rsplit(canon_spaces(root), " ")
for i=1, #words do
if (word_idx == options["wordct"]) then
local rdp_word = words[i]
if syllcount == "all" then
if (options["d-r"] == nil) then
options["d-r"] = "d"
end
rdp_word = d_r_adjust_root(words[i], words[i], options["d-r"])
words[i] = words[i] .. "-" .. rdp_word
elseif tonumber(syllable_count) ~= nil then
syllable_count = tonumber(syllable_count)
if(options["phonetic"]) then
rdp_word = ulower(options["phonetic"])
options["add_hyphen"] = true
end
local rdp_syllabification = export.syllabify_and_align(rsub(rdp_word, "(" .. V .. ")" .. "(" .. V .. ")", "%1.%2"))
rdp_syllabification = rsub(rdp_syllabification, "[-]([^ ])", "-.%1")
rdp_syllabification = rsplit(rdp_syllabification, "%.")
local rdp_add = ""
for j=1, #rdp_syllabification do
local rdp_syll = rdp_syllabification[j]
if j > syllable_count then
break
elseif j == syllable_count then
if syllable_count ~= #rdp_syllabification or #rdp_syllabification == 1 then
if rdp_syll:sub(-1) ~= "-" or syllable_count == 1 then
rdp_syll = rsub(rdp_syll, "(".. V .. ").*", "%1")
end
rdp_syll = rsub(rdp_syll, "-$", "")
if options["keep_cluster"] ~= true then
rdp_syll = rsub(rdp_syll, "(" .. C .. "+)(".. V .. ")",
function(consonants, vowel)
if not m_table.contains({"ng", "ts", "ch", "sh"}, consonants)
and not rmatch(consonants, "[dnt]y") then
consonants = consonants:sub(1,1)
end
return consonants .. vowel
end
)
end
end
end
rdp_add = rdp_add .. rdp_syll
end
if options["add_hyphen"] or syllable_count > 1 or rdp_word ~= ulower(rdp_word) then
rdp_add = rdp_add .. "-"
if (options["d-r"] == nil) then
options["d-r"] = "d"
end
end
if syllable_count == 1 then
rdp_add = ulower(rdp_add)
end
if(options["phonetic"]) then
words[i] = rdp_add .. words[i]
else
rdp_word = d_r_adjust_root(rdp_add, rdp_word, options["d-r"])
words[i] = rdp_add .. rdp_word
end
else
error('Syllable count must be numerical or "all".')
end
end
word_idx = word_idx + 1
end
words = table.concat(words, " ")
words = export.remove_accents(words)
return words
end
-- Pang-angkop/Linker
--- Add the appropriate Cebuano linker (-ng or na) to the end of a phrase.
-- @param text string Input text to which the linker is attached.
-- @param consider_case boolean|nil If true, respects capitalization for the linker.
-- @return string Text with the correct linker appended.
function export.add_linker(text, consider_case)
if text == nil or type(text) ~= "string" then
error("There should be an input parameter.")
end
local orig_text = export.remove_accents(text)
local input_text = ulower(orig_text)
local last_letter = rmatch(text, "(.)$")
local is_last_caps = ulower(last_letter) ~= last_letter
if(rfind(input_text, "[" .. vowel .. "n]$")) and (not is_last_caps or consider_case) then
-- Multi-word checker, always add to the last word
local words = rsplit(input_text, "[ %-]")
local options = {
wordct = #words
}
input_text = export.add_affix(input_text, "-ng", options)
else
input_text = input_text .. " na"
end
-- Fix capitalization with what already exists
local linker = input_text:sub(#orig_text + 1)
input_text = orig_text .. ((is_last_caps and consider_case) and uupper(linker) or linker)
return input_text
end
--- Deconstructs a Cebuano word into root + affix analysis.
-- Tries to peel off known circumfixes, prefixes, suffixes, infixes, and
-- reduplication layers in a fixed order, optionally guided by:
-- - a dictionary root hint (`options.root`)
-- - a restricted affix set (`options.affix`), e.g. "mag-+pa-+rdp".
--
-- The function records:
-- - root(s)
-- - circumfixes, prefixes, infixes, suffixes
-- - reduplication type(s)
-- - whether nasal assimilation, syncope, or metathesis were detected.
--
-- @param word string Surface word to analyze (UTF-8).
-- @param options table|nil Optional settings:
-- options.root string|nil Root hint (dictionary form) to guide validity checks.
-- options.affix string|nil Plus-separated affix hints, e.g. "mag-+pa-+rdp".
-- Use "rdp" or "reduplication" to force a reduplication check.
-- @return table analysis Table with fields:
-- analysis.root string[] An array containing the final analyzed root.
-- analysis.circumfix string[] List of circumfixes found ("ka- -an", etc.).
-- analysis.prefix string[] List of prefixes found ("mag-", "pa-", ...).
-- analysis.infix string[] List of infixes found ("-um-", "-in-", ...).
-- analysis.suffix string[] List of suffixes found ("-an", "-in", ...).
-- analysis.reduplication string[] List of reduplication types ("initial", ...).
-- analysis.nasal_assimilation boolean[] Per-step nasal assimilation flags.
-- analysis.syncope boolean[] Per-step syncope flags.
-- analysis.metathesis boolean[] Per-step metathesis flags.
-- TODO: mag- pa- ka- consecutively should be magpaka-
-- TODO: Fix if no root was provided
function export.deconstruct(word, options)
if options == nil then
options = {}
elseif type(options) ~= "table" then
error("Options parameter must be a table or nil.")
end
local test_root = options["root"] or nil
if test_root then
test_root = export.remove_accents(ulower(test_root))
else
error("export.deconstruct() currently requires options.root") -- Require root for now
end
local test_affix_list = options["affix"] or nil
if test_affix_list then
test_affix_list = canon_spaces(test_affix_list)
test_affix_list = rsub(test_affix_list, "([ ]?)+([ ]?)", "+")
test_affix_list = rsplit(test_affix_list, "+")
end
local found_analysis = {
["root"] = {},
["circumfix"] = {},
["prefix"] = {},
["infix"] = {},
["suffix"] = {},
["reduplication"] = {},
["nasal_assimilation"] = {},
["syncope"] = {},
["metathesis"] = {}
}
local last_action = nil
local last_affix = nil
local affix_list = ceb_affix_list
-- Break down affixes in this order
local op_order = {"circumfix", "prefix_infix", "prefix_multi", "reduplication", "prefix_priority", "suffix", "prefix_single", "infix"}
local root_word = word
--- Normalizes semivowel spellings in a word.
-- Collapses sequences like "iyV" → "yV" and "uwV" → "wV"
-- to make matching roots more robust.
-- @param word string The word to normalize. Must be a UTF-8 string.
-- @return string The normalized word.
local function normalize_semivowels(word)
local new_word = word
new_word = rsub(new_word, "iy(" .. V .. ")" , "y%1")
new_word = rsub(new_word, "uw(" .. V .. ")", "w%1")
return new_word
end
--- Tests if a candidate string is a plausible root.
-- Uses a hint root when available, allowing for regular sound alternations:
-- - i/e, u/o alternations
-- - d/r alternation
-- - nasal assimilation (m/n/ng before certain consonants)
-- If no root hint is given, falls back to a simple heuristic:
-- the candidate must contain at least two vowels.
-- Also rejects certain pang- forms without explicit nasal assimilation info.
-- @param word string Candidate root (possibly with assimilation).
-- @param find_root string|nil Root hint to match against (de-accented).
-- @param affix string|nil Affix being removed (used to detect pang-/pang- + vowel cases).
-- @param nasal_assimilation boolean|nil True if analysis already assumes nasal assimilation.
-- @return boolean true if the candidate looks like a valid root, false otherwise.
local function is_valid_root(word, find_root, affix, nasal_assimilation)
local word = word and ulower(rsub(word, "ng", "ŋ"))
local find_root = find_root and ulower(rsub(find_root, "ng", "ŋ"))
local affix = affix and ulower(rsub(affix, "ng", "ŋ"))
-- Normalize y/iy and w/uw
word = normalize_semivowels(word)
find_root = normalize_semivowels(find_root)
if word and affix and rfind(word, "^[aeiou]") and rmatch(affix, "([mŋ])-$") and not nasal_assimilation then
return false -- Roots that do have a correct pang- affix, have the root starting with hyphen during check
end
if find_root and #find_root > 0 then
local find_root = ulower(find_root)
local j = 1
for i = 1, #word do
local word_char = word:sub(i,i)
local find_root_char = find_root:sub(j,j)
if (word_char == find_root_char)
or (word_char == "i" and find_root_char == "e")
or (word_char == "u" and find_root_char == "o")
or (word_char == "r" and find_root_char == "d")
or (word_char == "n" and rfind(find_root_char, "[dst]"))
or (word_char == "m" and rfind(find_root_char, "[bp]"))
or (word_char == "ŋ" and rfind(find_root_char, "[kg]"))
then
j = j+1
if j > #find_root then
return true
end
end
end
return false
else
local _, count = word:gsub(V, "")
return count >= 2
end
end
--- Checks and repairs root syncopation caused by native verb suffixes.
-- Handles patterns like:
-- - CVCV(C) + native suffix → syncoped surface base
-- - roots with /h/ and vowel alternations (e.g. kuha → kunan / kuhanan)
-- - consonant-final roots with two-consonant endings and metathesis.
-- If the candidate `root` appears to be a syncoped form of `find_root`,
-- rewrites it back to a more canonical/root-like form and flags syncope / metathesis.
-- @param root string Candidate root (already stripped of the suffix).
-- @param clean_affix string Suffix being removed, without hyphens (e.g. "an", "hin").
-- @param find_root string The expected dictionary root used as a reference.
-- @return table result Table with:
-- result.root string Possibly corrected root.
-- result.syncope boolean True if syncopation was detected.
-- result.metathesis boolean True if metathesis was detected.
local function check_for_syncope(root, clean_affix, find_root)
local ceb_native_verb_suffixes = {
"han", "hon", "hi", "ha",
"an", "on", "i", "a",
"nan", "non", "ni", "na"
}
local result = {
["root"] = root,
["syncope"] = false,
["metathesis"] = false
}
-- Helper: consonant assimilation fixer
local function apply_disassimilation(root, sync_root)
root = rsub(root, "r(" .. C .. ")$", "d%1")
root = rsub(root, "r$", "d")
local rv1, rc1, rc2 = rmatch(root, "(" .. V .. ")(" .. C .. ")(" .. C .. ")$")
local sv1, sc1, sc2 = rmatch(sync_root, "(" .. V .. ")(" .. C .. ")(" .. C .. ")$")
if not (rv1 and sv1) then return root end
if rv1 == sv1 then
-- [dst]+n → [dst]+ŋ
if rc1 == sc1 and rfind(rc1, "[dst]") and rc2 == "n" and sc2 == "ŋ" then
return rsub(root, rv1 .. rc1 .. rc2 .. "$", rv1 .. rc1 .. sc2)
end
-- m+[bp] → n/ŋ+[bp]
if rc2 == sc2 and rc1 == "m" and rfind(sc1, "[nŋ]") and rfind(rc2, "[bp]") then
return rsub(root, rv1 .. rc1 .. rc2 .. "$", rv1 .. sc1 .. rc2)
end
-- n+[dst] → ŋ+[dst]
if rc2 == sc2 and rc1 == "n" and sc1 == "ŋ" and rfind(rc2, "[dst]") then
return rsub(root, rv1 .. rc1 .. rc2 .. "$", rv1 .. sc1 .. rc2)
end
-- ŋ+[gk] → n+[gk]
if rc2 == sc2 and rc1 == "ŋ" and sc1 == "n" and rfind(rc2, "[gk]") then
return rsub(root, rv1 .. rc1 .. rc2 .. "$", rv1 .. sc1 .. rc2)
end
end
return root
end
-- Check if suffix triggers syncopation rules
local is_native_suffix = m_table.contains(ceb_native_verb_suffixes, clean_affix)
local root_matched = rfind(root, find_root)
if is_native_suffix and not root_matched then
local syncoped_root, syncoped_meta_root, end_root_stem
local may_have_syncope = false
if rfind(find_root, "ah" .. V .. "$") then
may_have_syncope = rfind(root, V .. "$")
end_root_stem = rsub(find_root, "^.*ah(" .. V .. ")$", "ah%1")
syncoped_root = rsub(find_root, "^.*ah(" .. V .. ")$", "%1")
elseif rfind(find_root, "ah" .. V .. C .."$") then
may_have_syncope = rfind(root, V .. C .. "$")
end_root_stem = rsub(find_root, "^.*ah(" .. V .. ")(" .. C .. ")$", "ah%1%2")
syncoped_root = rsub(find_root, "^.*ah(" .. V .. ")(" .. C .. ")$", "%1%2")
elseif rfind(find_root, V .. "h" .. V .. "$") then
-- Case: vowel-final root with h (e.g., "kuha" → "kunan" → "kuhanan")
may_have_syncope = rfind(root, V .. "$")
end_root_stem = rsub(find_root, "^.*(" .. V .. ")h(" .. V .. ")$", "%1h%2")
syncoped_root = rsub(find_root, "^.*(" .. V .. ")h(" .. V .. ")$", "%1")
elseif rfind(find_root, V .. "h" .. V .. C .."$") then
-- Case: consonant-final root with h (e.g., "buhos" → "busan" → "buhusan")
may_have_syncope = rfind(root, V .. C .. "$")
end_root_stem = rsub(find_root, "^.*(" .. V .. ")h(" .. V .. ")(" .. C .. ")$", "%1h%2%3")
syncoped_root = rsub(find_root, "^.*(" .. V .. ")h(" .. V .. ")(" .. C .. ")$", "%1%3")
elseif rfind(find_root, V .. "$") then
-- Case: vowel-final root (e.g., "hingi" → "hing(-)an" → "hingian")
may_have_syncope = rfind(root, V .. C .. "$")
end_root_stem = rsub(find_root, "^.*(" .. V .. ")(" .. C .. ")(" .. V .. ")$", "%1%2%3")
syncoped_root = rsub(find_root, "^.*(" .. V .. ")(" .. C .. ")(" .. V .. ")$", "%1%2")
else
-- Case: consonant-final root (two-consonant endings)
may_have_syncope = rfind(root, V .. C .. C .. "$")
end_root_stem = rsub(find_root, "^.*(" .. V .. ")(" .. C .. ")(" .. V .. ")(" .. C .. ")$", "%1%2%3%4")
syncoped_root = rsub(find_root, "^.*(" .. V .. ")(" .. C .. ")(" .. V .. ")(" .. C .. ")$", "%1%2%4")
syncoped_meta_root = rsub(find_root, "^.*(" .. V .. ")(" .. C .. ")(" .. V .. ")(" .. C .. ")$", "%1%4%2")
if may_have_syncope and syncoped_root then
root = apply_disassimilation(root, syncoped_root)
end
end
-- Detect and correct syncopation or metathesis
if may_have_syncope then
if syncoped_root and rfind(root, syncoped_root .. "$") then
result.root = rsub(root, syncoped_root .. "$", end_root_stem)
result.syncope = true
end
if syncoped_meta_root and rfind(root, syncoped_meta_root .. "$") then
result.root = rsub(root, syncoped_meta_root .. "$", end_root_stem)
result.syncope = true
result.metathesis = true
end
end
end
return result
end
--- Attempts to strip a single affix from a word.
-- Supports:
-- - circumfixes: "prefix ... suffix" (e.g. ka- -an)
-- - prefixes: pag-, mag-, etc., with optional nasal assimilation reversal
-- - suffixes: -an, -in, -i, etc., with optional syncope correction
-- - infixes: -um-, -in-, etc.
-- Also detects and annotates nasal assimilation, syncope and metathesis when present.
-- @param word string The current surface form being analyzed.
-- @param affix string The affix pattern (with or without hyphens).
-- @param affix_type string One of "prefix", "suffix", "infix", or "circumfix".
-- @param find_root string|nil Root hint used for nasal assimilation and syncope checks.
-- @return table info Table with:
-- info.root string Resulting candidate root after removal attempt.
-- info.nasal_assimilation boolean true if nasal assimilation was reversed.
-- info.syncope boolean true if syncopation was detected/fixed.
-- info.metathesis boolean true if metathesis was detected/fixed.
local function try_remove_affix(word, affix, affix_type, find_root)
local remove_affix_test = {
["root"] = word,
["nasal_assimilation"] = false,
["syncope"] = false,
["metathesis"] = false,
}
local word = rsub(word, "ng", "ŋ")
local find_root = rsub(find_root, "ng", "ŋ")
local clean_affix = rsub(affix, "-", "")
clean_affix = rsub(clean_affix, "ng$", "ŋ")
if affix_type == "circumfix" then
local parts = rsplit(canon_spaces(clean_affix), " ")
remove_affix_test.root = rsub(word, "^" .. parts[1] .. "(.*)" .. parts[2] .. "$", "%1")
remove_affix_test.root = rsub(remove_affix_test.root, "-$", "")
elseif affix_type == "prefix" then
local root_first_cons, root_first_vowel
-- Check if prefix is ending with nasal character
local nasal_prefix_end = rmatch(clean_affix, "[ai]([mnŋ])$")
-- If prefix is possible to have nasal assimilation, try to have unassimilated word
if nasal_prefix_end and find_root then
-- Get root components
local root_first_cons, root_first_vowel = rmatch(find_root, "^(" .. C .. "*)(" .. V .. ")(.*)$")
if root_first_cons == "" then
root_first_cons = "ʔ"
end
if root_first_cons and root_first_vowel then
if (nasal_prefix_end == "m" and rfind(root_first_cons, "^[bp]"))
or (nasal_prefix_end == "n" and rfind(root_first_cons, "^[dst]"))
or (nasal_prefix_end == "ŋ" and rfind(root_first_cons, "^[kgʔ]"))
then
-- If consonants before vowel is a cluster, separate them
local const_start = mw.ustring.sub(root_first_cons, 1,1)
local const_other = mw.ustring.sub(root_first_cons, 2)
if rfind(word, "^(" .. clean_affix .. ")(" .. root_first_vowel .. ")")
or rfind(word, "^(" .. clean_affix .. ")(" .. const_other.. ")(" .. root_first_vowel .. ")")
then
remove_affix_test["nasal_assimilation"] = true
end
if const_other and const_other ~= "" then
word = rsub(
word,
"^(" .. clean_affix .. ")(" .. const_other.. ")(" .. root_first_vowel .. ")(" .. nasal_prefix_end ..")(" .. const_other.. ")%3(.*)$",
clean_affix .. const_start .. "%2%3" .. root_first_cons .. "%3%6"
)
word = rsub(
word,
"^(" .. clean_affix .. ")(" .. root_first_vowel .. ")(" .. nasal_prefix_end ..")(" .. const_other.. ")%2(.*)$",
clean_affix .. const_start .. "%2" .. root_first_cons .. "%2%5"
)
word = rsub(
word,
"^(" .. clean_affix .. ")(" .. const_other.. ")(" .. root_first_vowel .. ")(.*)$",
clean_affix .. root_first_cons .. "%3%4"
)
end
word = rsub(
word,
"^(" .. clean_affix .. ")(" .. root_first_vowel .. ")(" .. nasal_prefix_end ..")%2(.*)$",
clean_affix .. const_start .. "%2" .. root_first_cons .. "%2%4"
)
word = rsub(
word,
"^(" .. clean_affix .. ")(" .. root_first_vowel .. ")(.*)$",
clean_affix .. root_first_cons .. "%2%3"
)
word = rsub(word, "ʔ", "")
end
end
end
remove_affix_test.root = rsub(word, "^" .. clean_affix, "")
elseif affix_type == "suffix" then
-- Remove suffix and trailing dash
remove_affix_test.root = rsub(word, clean_affix .. "$", "")
remove_affix_test.root = rsub(remove_affix_test.root, "-$", "")
if remove_affix_test.root ~= word then
local syncope_check = check_for_syncope(remove_affix_test.root, clean_affix, find_root)
remove_affix_test.root = syncope_check.root
remove_affix_test.syncope = syncope_check.syncope
remove_affix_test.metathesis = syncope_check.metathesis
end
elseif affix_type == "infix" then
remove_affix_test.root = rsub(word, "^(" .. C .. "+)" .. clean_affix .. "(.*)$", "%1%2")
end
remove_affix_test.root = rsub(remove_affix_test.root, "ŋ", "ng")
return remove_affix_test
end
--- Attempts to strip reduplication from a word.
-- Handles initial, partial, full, and D/R or vowel alternation patterns, e.g.:
-- - CV- reduplication: bili → bi-bili-, etc.
-- - full reduplication: bili-bili, bili~bili with d/r or e/i, o/u alternations.
-- Returns the base form and reduplication type if a pattern matches.
-- @param word string Surface word to test for reduplication.
-- @return table rdp_result Table with:
-- rdp_result.root string|nil Root form if reduplication is detected, nil otherwise.
-- rdp_result.type string|nil One of "initial", "partial", or "full" when detected.
local function try_remove_reduplication(word)
local rdp_root, count
local rdp_result = {
["root"] = nil,
["type"] = nil
}
local rdp_tests = {
{"^d(" .. V .. ")r%1(.+)$", "d%1%2", "initial"}, -- D/R reduplication
{"^(" .. C .. ")(" .. V .. ")%1([lryw])%2(.+)$", "%1%3%2%4", "initial"}, -- CV reduplication with no cluster
{"^(" .. C .. "*" .. V .. ")%1(.+)$", "%1%2", "initial"}, -- CV reduplication with cluster
{"^d(.-)(" .. V .. ")%-r%1%2$", "d%1%2", "full"}, -- Full word reduplication with D/R alteration
{"^(.-)d%-%1r$", "%1d", "full"}, -- Full word reduplication with D/R suffix alteration
{"^(.-)e%-%1i$", "%1e", "full"}, -- Full word reduplication with E/I suffix alteration
{"^(.-)o%-%1u$", "%1o", "full"}, -- Full word reduplication with O/U suffix alteration
{"^(.-)%-%1$", "%1", "full"}, -- Full word reduplication with dash
{"^d(.-)(" .. V .. ")%-r%1%2", "d%1%2", "partial"}, -- Partial word reduplication with D/R alteration
{"^(.-)(" .. V .. ")%-%1%2", "%1%2", "partial"} -- Partial word reduplication with dash
}
for _, test in ipairs(rdp_tests) do
rdp_root, count = rsubn(word, test[1], test[2])
if count > 0 then
rdp_result.root = rdp_root
rdp_result.type = test[3]
break
end
end
return rdp_result
end
--- Normalizes the analyzed root against the given dictionary root.
-- This post-processes the final `root_word` so it better matches `test_root` by:
-- - repairing w/uw and y/iy differences after normalization
-- - adjusting final vowel alternations (i/e, u/o)
-- - adjusting final d/r alternation
-- - handling the special case where analyzed root starts with r- but
-- the dictionary root starts with d-.
-- @param root_word string The root as derived by deconstruction.
-- @param test_root string The dictionary root hint (de-accented).
-- @return string Normalized root that better matches the dictionary form.
local function normalize_analyzed_root(root_word, test_root)
local normalized_root = root_word
local wy_normalize_root = normalize_semivowels(normalized_root)
local wy_normalize_test = normalize_semivowels(test_root)
-- If they have the same length after normalization, then it's possible we need to adjust semivowels
if(#wy_normalize_root == #wy_normalize_test) then
for i = 1, #test_root do
local normalized_char = string.sub(wy_normalize_root, i, i)
local current_char = string.sub(test_root, i, i)
local next_char = string.sub(test_root, i+1, i+1)
if(normalized_char == "w" and current_char == "u" and next_char == "w") then
wy_normalize_root = string.sub(wy_normalize_root, 1, i - 1) .. "u" .. string.sub(wy_normalize_root, i)
end
if(normalized_char == "y" and current_char == "i" and next_char == "y") then
wy_normalize_root = string.sub(wy_normalize_root, 1, i - 1) .. "i" .. string.sub(wy_normalize_root, i)
end
end
normalized_root = wy_normalize_root
end
if #normalized_root == #test_root then
-- Get last vowel and consonant of given root and analyzed root for suffix fixes
local root_last_vowel, root_last_cons = rmatch(normalized_root, "(" .. V .. ")(" .. C .. "*)$")
local test_root_last_vowel, test_root_last_cons = rmatch(test_root, "(" .. V .. ")(" .. C .. "*)$")
if test_root_last_vowel == "o" and root_last_vowel == "u" or
test_root_last_vowel == "e" and root_last_vowel == "i" or
test_root_last_vowel == "i" and root_last_vowel == "e" then
normalized_root = rsub(normalized_root, "(" .. V .. ")(" .. C .. "*)$", test_root_last_vowel .. "%2")
end
if test_root_last_cons == "d" and root_last_cons == "r" then
normalized_root = rsub(normalized_root, "(" .. V .. ")(" .. C .. "*)$", "%1d")
end
if normalized_root:sub(2) == test_root:sub(2)
and normalized_root:sub(1,1) == "r"
and test_root:sub(1,1) == "d" then
normalized_root = test_root
end
end
return normalized_root
end
-- Start testing for affixes
while true do
local changed = false
if test_affix_list and #test_affix_list > 0 then
for _, affix in ipairs(test_affix_list) do
if affix == "rdp" or affix == "reduplication" then
local rdp_test = try_remove_reduplication(root_word)
if rdp_test.root and is_valid_root(rdp_test.root, test_root) then
table.insert(found_analysis.reduplication, rdp_test.type)
root_word = rdp_test.root
last_action = "reduplication"
changed = true
else
break
end
else
local affix_type = get_affix_type(affix)
if (affix_type == "" and rfind(affix, "- -")) then
affix_type = "circumfix" -- workaround for now
end
local candidate_root_test = try_remove_affix(root_word, affix, affix_type, test_root)
local candidate_root = candidate_root_test.root
if candidate_root ~= root_word then
if is_valid_root(candidate_root, test_root, affix, candidate_root_test.nasal_assimilation) then
local current_affix_list = found_analysis[affix_type]
table.insert(current_affix_list, affix)
last_affix = affix
root_word = rsub(candidate_root, "^-", "")
root_word = rsub(root_word, "-$", "")
changed = true
last_action = affix_type
if candidate_root_test.nasal_assimilation then
table.insert(found_analysis.nasal_assimilation, true)
end
if candidate_root_test.syncope then
table.insert(found_analysis.syncope, true)
end
if candidate_root_test.metathesis then
table.insert(found_analysis.metathesis, true)
end
else
break
end
end
end
end
else
for _, op_type in ipairs(op_order) do
local affixes = affix_list[op_type]
if op_type == "reduplication" then
local rdp_test = try_remove_reduplication(root_word)
if rdp_test.root and is_valid_root(rdp_test.root, test_root) then
table.insert(found_analysis.reduplication, rdp_test.type)
root_word = rdp_test.root
last_action = op_type
changed = true
break
end
-- Use prefix indexing to speed up prefix search especially for long words
elseif op_type == "prefix_multi" then
local first_letter = root_word:sub(1, 1)
affixes = prefix_multi_index[first_letter] or {}
elseif op_type == "prefix_single" then
local first_letter = root_word:sub(1, 1)
affixes = prefix_single_index[first_letter] or {}
end
op_type = rmatch(op_type, "^[^_]+")
-- Try to remove affix to test if valid root
if affixes then
for _, affix in ipairs(affixes) do
local candidate_root_test = try_remove_affix(root_word, affix, op_type, test_root)
local candidate_root = candidate_root_test.root
if candidate_root ~= root_word
and is_valid_root(candidate_root, test_root, affix, candidate_root_test.nasal_assimilation) then
-- i- merger
local current_affix_list = found_analysis[op_type]
if op_type == "prefix"
and last_action == "prefix"
and current_affix_list[#current_affix_list] == "i-"
and m_table.contains({ "pag-", "ka-", "pa-" }, affix) then
current_affix_list[#current_affix_list] = "i" .. affix
last_affix = current_affix_list[#current_affix_list]
elseif op_type == "prefix" and last_action == "suffix" and
m_table.contains(ceb_affix_list["circumfix"], affix .. " " .. last_affix) then
table.remove(found_analysis[last_action])
table.insert(found_analysis.circumfix, affix .. " " .. last_affix)
else
table.insert(current_affix_list, affix)
last_affix = affix
end
root_word = rsub(candidate_root, "^-", "")
root_word = rsub(root_word, "-$", "")
changed = true
last_action = op_type
if candidate_root_test.nasal_assimilation then
table.insert(found_analysis.nasal_assimilation, true)
end
if candidate_root_test.syncope then
table.insert(found_analysis.syncope, true)
end
if candidate_root_test.metathesis then
table.insert(found_analysis.metathesis, true)
end
break
end
end
end
if changed then break end
end
end
if not changed then break end
end
if test_root then
root_word = normalize_analyzed_root(root_word, test_root)
if rfind(root_word, "ng$") and rfind(test_root, "n$") and rfind(root_word, "^" .. test_root) then
root_word = test_root
table.insert(found_analysis.suffix, "-g")
end
end
table.insert(found_analysis.root, root_word)
return found_analysis
end
return export
