--[[============================================================
--=
--= LuaPreprocess v1.21-dev - preprocessing library
--= by Marcus 'ReFreezed' Thunström
--=
--= License: MIT (see the bottom of this file)
--= Website: http://refreezed.com/luapreprocess/
--= Documentation: http://refreezed.com/luapreprocess/docs/
--=
--= Tested with Lua 5.1, 5.2, 5.3, 5.4 and LuaJIT.
--=
--==============================================================
API:
Global functions in metaprograms:
- copyTable
- escapePattern
- getIndentation
- isProcessing
- pack
- pairsSorted
- printf
- readFile, writeFile, fileExists
- run
- sortNatural, compareNatural
- tokenize, newToken, concatTokens, removeUselessTokens, eachToken, isToken, getNextUsefulToken
- toLua, serialize, evaluate
Only during processing:
- getCurrentPathIn, getCurrentPathOut
- getOutputSoFar, getOutputSoFarOnLine, getOutputSizeSoFar, getCurrentLineNumberInOutput, getCurrentIndentationInOutput
- loadResource, callMacro
- outputValue, outputLua, outputLuaTemplate
- startInterceptingOutput, stopInterceptingOutput
Macros:
- ASSERT
- LOG
Search this file for 'EnvironmentTable' and 'PredefinedMacros' for more info.
Exported stuff from the library:
- (all the functions above)
- VERSION
- metaEnvironment
- processFile, processString
Search this file for 'ExportTable' for more info.
----------------------------------------------------------------
How to metaprogram:
The exclamation mark (!) is used to indicate what code is part of
the metaprogram. There are 4 main ways to write metaprogram code:
!... The line will simply run during preprocessing. The line can span multiple actual lines if it contains brackets.
!!... The line will appear in both the metaprogram and the final program. The line must be an assignment.
!(...) The result of the parenthesis will be outputted as a literal if it's an expression, otherwise it'll just run.
!!(...) The result of the expression in the parenthesis will be outputted as Lua code. The result must be a string.
Short examples:
!if not isDeveloper then
sendTelemetry()
!end
!!local tau = 2*math.pi -- The expression will be evaluated in the metaprogram and the result will appear in the final program as a literal.
local bigNumber = !(5^10)
local font = !!(isDeveloper and "loadDevFont()" or "loadUserFont()")
-- See the full documentation for additional features (like macros):
-- http://refreezed.com/luapreprocess/docs/extra-functionality/
----------------------------------------------------------------
-- Example program:
-- Normal Lua.
local n = 0
doTheThing()
-- Preprocessor lines.
local n = 0
!if math.random() < 0.5 then
n = n+10 -- Normal Lua.
-- Note: In the final program, this will be in the
-- same scope as 'local n = 0' here above.
!end
!for i = 1, 3 do
print("3 lines with print().")
!end
-- Extended preprocessor line. (Lines are consumed until brackets
-- are balanced when the end of the line has been reached.)
!newClass{ -- Starts here.
name = "Entity",
props = {x=0, y=0},
} -- Ends here.
-- Preprocessor block.
!(
local dogWord = "Woof "
function getDogText()
return dogWord:rep(3)
end
)
-- Preprocessor inline block. (Expression that returns a value.)
local text = !("The dog said: "..getDogText())
-- Preprocessor inline block variant. (Expression that returns a Lua code string.)
_G.!!("myRandomGlobal"..math.random(5)) = 99
-- Dual code (both preprocessor line and final output).
!!local partial = "Hello"
local whole = partial .. !(partial..", world!")
print(whole) -- HelloHello, world!
-- Beware in preprocessor blocks that only call a single function!
!( func() ) -- This will bee seen as an inline block and output whatever value func() returns as a literal.
!( func(); ) -- If that's not wanted then a trailing `;` will prevent that. This line won't output anything by itself.
-- When the full metaprogram is generated, `!(func())` translates into `outputValue(func())`
-- while `!(func();)` simply translates into `func();` (because `outputValue(func();)` would be invalid Lua code).
-- Though in this specific case a preprocessor line (without the parenthesis) would be nicer:
!func()
-- For the full documentation, see:
-- http://refreezed.com/luapreprocess/docs/
--============================================================]]
local PP_VERSION = "1.21.0-dev"
local MAX_DUPLICATE_FILE_INSERTS = 1000 -- @Incomplete: Make this a parameter for processFile()/processString().
local MAX_CODE_LENGTH_IN_MESSAGES = 60
local KEYWORDS = {
"and",
"break",
"do",
"else",
"elseif",
"end",
"false",
"for",
"function",
"if",
"in",
"local",
"nil",
"not",
"or",
"repeat",
"return",
"then",
"true",
"until",
"while",
-- Lua 5.2
"goto", -- @Incomplete: A parameter to disable this for Lua 5.1?
}
for i, v in ipairs(KEYWORDS) do
KEYWORDS[v], KEYWORDS[i] = true, nil
end
local PREPROCESSOR_KEYWORDS = {
"file",
"insert",
"line",
}
for i, v in ipairs(PREPROCESSOR_KEYWORDS) do
PREPROCESSOR_KEYWORDS[v], PREPROCESSOR_KEYWORDS[i] = true, nil
end
local PUNCTUATION = {
"+",
"-",
"*",
"/",
"%",
"^",
"#",
"==",
"~=",
"<=",
">=",
"<",
">",
"=",
"(",
")",
"{",
"}",
"[",
"]",
";",
":",
",",
".",
"..",
"...",
-- Lua 5.2
"::",
-- Lua 5.3
"//",
"&",
"|",
"~",
">>",
"<<",
}
for i, v in ipairs(PUNCTUATION) do
PUNCTUATION[v], PUNCTUATION[i] = true, nil
end
local ESCAPE_SEQUENCES_EXCEPT_QUOTES = {
["\a"] = [[\a]],
["\b"] = [[\b]],
["\f"] = [[\f]],
["\n"] = [[\n]],
["\r"] = [[\r]],
["\t"] = [[\t]],
["\v"] = [[\v]],
["\\"] = [[\\]],
}
local ESCAPE_SEQUENCES = {
['"'] = [[\"]],
["'"] = [[\']],
}
for k, v in pairs(ESCAPE_SEQUENCES_EXCEPT_QUOTES) do
ESCAPE_SEQUENCES[k] = v
end
local USELESS_TOKENS = { whitespace = true, comment = true }
local LOG_LEVELS = {
["off"] = 0,
["error"] = 1,
["warning"] = 2,
["info"] = 3,
["debug"] = 4,
["trace"] = 5,
}
local metaEnv = nil
local dummyEnv = {}
-- Controlled by processFileOrString():
local current_parsingAndMeta_isProcessing = false
local current_parsingAndMeta_isDebug = false
-- Controlled by _processFileOrString():
local current_anytime_isRunningMeta = false
local current_anytime_pathIn = ""
local current_anytime_pathOut = ""
local current_anytime_fastStrings = false
local current_parsing_insertCount = 0
local current_parsingAndMeta_onInsert = nil
local current_parsingAndMeta_resourceCache = nil
local current_parsingAndMeta_addLineNumbers = false
local current_parsingAndMeta_macroPrefix = ""
local current_parsingAndMeta_macroSuffix = ""
local current_parsingAndMeta_strictMacroArguments = true
local current_meta_pathForErrorMessages = ""
local current_meta_output = nil -- Top item in current_meta_outputStack.
local current_meta_outputStack = nil
local current_meta_canOutputNil = true
local current_meta_releaseMode = false
local current_meta_maxLogLevel = "trace"
local current_meta_locationTokens = nil
--==============================================================
--= Local Functions ============================================
--==============================================================
local assertarg
local countString, countSubString
local getLineNumber
local loadLuaString
local maybeOutputLineNumber
local sortNatural
local tableInsert, tableRemove, tableInsertFormat
local utf8GetCodepointAndLength
local F = string.format
local function tryToFormatError(err0)
local err, path, ln = nil
if type(err0) == "string" then
do
path, ln, err = err0:match("^(%a:[%w_/\\.]+):(%d+): (.*)")
if not err then
path, ln, err = err0:match("^([%w_/\\.]+):(%d+): (.*)")
if not err then
path, ln, err = err0:match("^(%S-):(%d+): (.*)")
end
end
end
end
if err then
return F("Error @ %s:%s: %s", path, ln, err)
else
return "Error: " .. tostring(err0)
end
end
local function printf(s, ...)
print(F(s, ...))
end
-- printTokens( tokens [, filterUselessTokens ] )
local function printTokens(tokens, filter)
for i, tok in ipairs(tokens) do
if not (filter and USELESS_TOKENS[tok.type]) then
printf("%d %-12s '%s'", i, tok.type, (F("%q", tostring(tok.value)):sub(2, -2):gsub("\\\n", "\\n")))
end
end
end
local function printError(s)
io.stderr:write(s, "\n")
end
local function printfError(s, ...)
printError(F(s, ...))
end
-- message = formatTraceback( [ level=1 ] )
local function formatTraceback(level)
local buffer = {}
tableInsert(buffer, "stack traceback:\n")
level = 1 + (level or 1)
local stack = {}
while level < 1 / 0 do
local info = debug.getinfo(level, "nSl")
if not info then
break
end
local isFile = info.source:find("^@") ~= nil
local sourceName = (isFile and info.source:sub(2) or info.short_src)
local subBuffer = { "\t" }
tableInsertFormat(subBuffer, "%s:", sourceName)
if info.currentline > 0 then
tableInsertFormat(subBuffer, "%d:", info.currentline)
end
if (info.name or "") ~= "" then
tableInsertFormat(subBuffer, " in '%s'", info.name)
elseif info.what == "main" then
tableInsert(subBuffer, " in main chunk")
elseif info.what == "C" or info.what == "tail" then
tableInsert(subBuffer, " ?")
else
tableInsertFormat(subBuffer, " in <%s:%d>", sourceName:gsub("^.*[/\\]", ""), info.linedefined)
end
tableInsert(stack, table.concat(subBuffer))
level = level + 1
end
while stack[#stack] == "\t[C]: ?" do
stack[#stack] = nil
end
for _, s in ipairs(stack) do
tableInsert(buffer, s)
tableInsert(buffer, "\n")
end
return table.concat(buffer)
end
-- printErrorTraceback( message [, level=1 ] )
local function printErrorTraceback(message, level)
printError(tryToFormatError(message))
printError(formatTraceback(1 + (level or 1)))
end
-- debugExit( )
-- debugExit( messageValue )
-- debugExit( messageFormat, ... )
local function debugExit(...)
if select("#", ...) > 1 then
printfError(...)
elseif select("#", ...) == 1 then
printError(...)
end
os.exit(2)
end
-- errorf( [ level=1, ] string, ... )
local function errorf(sOrLevel, ...)
if type(sOrLevel) == "number" then
error(F(...), (sOrLevel == 0 and 0 or 1 + sOrLevel))
else
error(F(sOrLevel, ...), 2)
end
end
-- local function errorLine(err) -- Unused.
-- if type(err) ~= "string" then error(err) end
-- error("\0"..err, 0) -- The 0 tells our own error handler not to print the traceback.
-- end
local function errorfLine(s, ...)
errorf(0, (current_parsingAndMeta_isProcessing and "\0" or "") .. s, ...) -- The \0 tells our own error handler not to print the traceback.
end
-- errorOnLine( path, lineNumber, agent=nil, s, ... )
local function errorOnLine(path, ln, agent, s, ...)
s = F(s, ...)
if agent then
errorfLine("%s:%d: [%s] %s", path, ln, agent, s)
else
errorfLine("%s:%d: %s", path, ln, s)
end
end
local errorInFile, runtimeErrorInFile
do
local function findStartOfLine(s, pos, canBeEmpty)
while pos > 1 do
if
s:byte(pos - 1) == 10--[[\n]]
and (
canBeEmpty or s:byte(pos) ~= 10--[[\n]]
)
then
break
end
pos = pos - 1
end
return math.max(pos, 1)
end
local function findEndOfLine(s, pos)
while pos < #s do
if
s:byte(pos + 1) == 10--[[\n]]
then
break
end
pos = pos + 1
end
return math.min(pos, #s)
end
local function _errorInFile(level, contents, path, pos, agent, s, ...)
s = F(s, ...)
pos = math.min(math.max(pos, 1), #contents + 1)
local ln = getLineNumber(contents, pos)
local lineStart = findStartOfLine(contents, pos, true)
local lineEnd = findEndOfLine(contents, pos - 1)
local linePre1Start = findStartOfLine(contents, lineStart - 1, false)
local linePre1End = findEndOfLine(contents, linePre1Start - 1)
local linePre2Start = findStartOfLine(contents, linePre1Start - 1, false)
local linePre2End = findEndOfLine(contents, linePre2Start - 1)
-- printfError("pos %d | lines %d..%d, %d..%d, %d..%d", pos, linePre2Start,linePre2End+1, linePre1Start,linePre1End+1, lineStart,lineEnd+1) -- DEBUG
errorOnLine(
path,
ln,
agent,
"%s\n>\n%s%s%s>-%s^%s",
s,
(linePre2Start < linePre1Start and linePre2Start <= linePre2End)
and F("> %s\n", (contents:sub(linePre2Start, linePre2End):gsub("\t", " ")))
or "",
(linePre1Start < lineStart and linePre1Start <= linePre1End)
and F("> %s\n", (contents:sub(linePre1Start, linePre1End):gsub("\t", " ")))
or "",
(lineStart <= lineEnd) and F("> %s\n", (contents:sub(lineStart, lineEnd):gsub("\t", " "))) or ">\n",
("-"):rep(pos - lineStart + 3 * countSubString(contents, lineStart, lineEnd, "\t", true)),
(level and "\n" .. formatTraceback(1 + level) or "")
)
end
-- errorInFile( contents, path, pos, agent, s, ... )
--[[local]]
function errorInFile(...)
_errorInFile(nil, ...)
end
-- runtimeErrorInFile( level, contents, path, pos, agent, s, ... )
--[[local]]
function runtimeErrorInFile(level, ...)
_errorInFile(1 + level, ...)
end
end
-- errorAtToken( token, position=token.position, agent, s, ... )
local function errorAtToken(tok, pos, agent, s, ...)
-- printErrorTraceback("errorAtToken", 2) -- DEBUG
errorInFile(current_parsingAndMeta_resourceCache[tok.file], tok.file, (pos or tok.position), agent, s, ...)
end
-- errorAfterToken( token, agent, s, ... )
local function errorAfterToken(tok, agent, s, ...)
-- printErrorTraceback("errorAfterToken", 2) -- DEBUG
errorInFile(
current_parsingAndMeta_resourceCache[tok.file],
tok.file,
tok.position + #tok.representation,
agent,
s,
...
)
end
-- runtimeErrorAtToken( level, token, position=token.position, agent, s, ... )
local function runtimeErrorAtToken(level, tok, pos, agent, s, ...)
-- printErrorTraceback("runtimeErrorAtToken", 2) -- DEBUG
runtimeErrorInFile(
1 + level,
current_parsingAndMeta_resourceCache[tok.file],
tok.file,
(pos or tok.position),
agent,
s,
...
)
end
-- internalError( [ message|value ] )
local function internalError(message)
message = message and " (" .. tostring(message) .. ")" or ""
error("Internal error." .. message, 2)
end
local function cleanError(err)
if type(err) == "string" then
err = err:gsub("%z", "")
end
return err
end
local function formatCodeForShortMessage(lua)
lua = lua:gsub("^%s+", ""):gsub("%s+$", ""):gsub("%s+", " ")
if #lua > MAX_CODE_LENGTH_IN_MESSAGES then
lua = lua:sub(1, MAX_CODE_LENGTH_IN_MESSAGES / 2) .. "..." .. lua:sub(-MAX_CODE_LENGTH_IN_MESSAGES / 2)
end
return lua
end
local ERROR_UNFINISHED_STRINGLIKE = 1
local function parseStringlikeToken(s, ptr)
local reprStart = ptr
local reprEnd
local valueStart
local valueEnd
local longEqualSigns = s:match("^%[(=*)%[", ptr)
local isLong = longEqualSigns ~= nil
-- Single line.
if not isLong then
valueStart = ptr
local i = s:find("\n", ptr, true)
if not i then
reprEnd = #s
valueEnd = #s
ptr = reprEnd + 1
else
reprEnd = i
valueEnd = i - 1
ptr = reprEnd + 1
end
-- Multiline.
else
ptr = ptr + 1 + #longEqualSigns + 1
valueStart = ptr
local i1, i2 = s:find("]" .. longEqualSigns .. "]", ptr, true)
if not i1 then
return nil, ERROR_UNFINISHED_STRINGLIKE
end
reprEnd = i2
valueEnd = i1 - 1
ptr = reprEnd + 1
end
local repr = s:sub(reprStart, reprEnd)
local v = s:sub(valueStart, valueEnd)
local tok = { type = "stringlike", representation = repr, value = v, long = isLong }
return tok, ptr
end
local NUM_HEX_FRAC_EXP = ("^( 0[Xx] (%x*) %.(%x+) [Pp]([-+]?%x+) )"):gsub(" +", "")
local NUM_HEX_FRAC = ("^( 0[Xx] (%x*) %.(%x+) )"):gsub(" +", "")
local NUM_HEX_EXP = ("^( 0[Xx] (%x+) %.? [Pp]([-+]?%x+) )"):gsub(" +", "")
local NUM_HEX = ("^( 0[Xx] %x+ %.? )"):gsub(" +", "")
local NUM_DEC_FRAC_EXP = ("^( %d* %.%d+ [Ee][-+]?%d+ )"):gsub(" +", "")
local NUM_DEC_FRAC = ("^( %d* %.%d+ )"):gsub(" +", "")
local NUM_DEC_EXP = ("^( %d+ %.? [Ee][-+]?%d+ )"):gsub(" +", "")
local NUM_DEC = ("^( %d+ %.? )"):gsub(" +", "")
-- tokens = _tokenize( luaString, path, allowPreprocessorTokens, allowBacktickStrings, allowJitSyntax )
local function _tokenize(s, path, allowPpTokens, allowBacktickStrings, allowJitSyntax)
s = s:gsub("\r", "") -- Normalize line breaks. (Assume the input is either "\n" or "\r\n".)
local tokens = {}
local ptr = 1
local ln = 1
while ptr <= #s do
local tok
local tokenPos = ptr
-- Whitespace.
if s:find("^%s", ptr) then
local i1, i2, whitespace = s:find("^(%s+)", ptr)
ptr = i2 + 1
tok = { type = "whitespace", representation = whitespace, value = whitespace }
-- Identifier/keyword.
elseif s:find("^[%a_]", ptr) then
local i1, i2, word = s:find("^([%a_][%w_]*)", ptr)
ptr = i2 + 1
if KEYWORDS[word] then
tok = { type = "keyword", representation = word, value = word }
else
tok = { type = "identifier", representation = word, value = word }
end
-- Number (binary).
elseif s:find("^0b", ptr) then
if not allowJitSyntax then
errorInFile(s, path, ptr, "Tokenizer", "Encountered binary numeral. (Feature not enabled.)")
end
local i1, i2, numStr = s:find("^(..[01]+)", ptr)
-- @Copypaste from below.
if not numStr then
errorInFile(s, path, ptr, "Tokenizer", "Malformed number.")
end
local numStrFallback = numStr
do
if s:find("^[Ii]", i2 + 1) then -- Imaginary part of complex number.
numStr = s:sub(i1, i2 + 1)
i2 = i2 + 1
elseif s:find("^[Uu][Ll][Ll]", i2 + 1) then -- Unsigned 64-bit integer.
numStr = s:sub(i1, i2 + 3)
i2 = i2 + 3
elseif s:find("^[Ll][Ll]", i2 + 1) then -- Signed 64-bit integer.
numStr = s:sub(i1, i2 + 2)
i2 = i2 + 2
end
end
local n = tonumber(numStr) or tonumber(numStrFallback) or tonumber(numStrFallback:sub(3), 2)
if not n then
errorInFile(s, path, ptr, "Tokenizer", "Invalid number.")
end
if s:find("^[%w_]", i2 + 1) then
-- This is actually not an error in Lua 5.2 and 5.3. Maybe we should issue a warning instead of an error here?
errorInFile(s, path, i2 + 1, "Tokenizer", "Malformed number.")
end
ptr = i2 + 1
tok = { type = "number", representation = numStrFallback, value = n }
-- Number.
elseif s:find("^%.?%d", ptr) then
local pat, maybeInt, lua52Hex, i1, i2, numStr = NUM_HEX_FRAC_EXP, false, true, s:find(NUM_HEX_FRAC_EXP, ptr)
if not i1 then
pat, maybeInt, lua52Hex, i1, i2, numStr = NUM_HEX_FRAC, false, true, s:find(NUM_HEX_FRAC, ptr)
if not i1 then
pat, maybeInt, lua52Hex, i1, i2, numStr = NUM_HEX_EXP, false, true, s:find(NUM_HEX_EXP, ptr)
if not i1 then
pat, maybeInt, lua52Hex, i1, i2, numStr = NUM_HEX, true, false, s:find(NUM_HEX, ptr)
if not i1 then
pat, maybeInt, lua52Hex, i1, i2, numStr =
NUM_DEC_FRAC_EXP, false, false, s:find(NUM_DEC_FRAC_EXP, ptr)
if not i1 then
pat, maybeInt, lua52Hex, i1, i2, numStr =
NUM_DEC_FRAC, false, false, s:find(NUM_DEC_FRAC, ptr)
if not i1 then
pat, maybeInt, lua52Hex, i1, i2, numStr =
NUM_DEC_EXP, false, false, s:find(NUM_DEC_EXP, ptr)
if not i1 then
pat, maybeInt, lua52Hex, i1, i2, numStr =
NUM_DEC, true, false, s:find(NUM_DEC, ptr)
end
end
end
end
end
end
end
if not numStr then
errorInFile(s, path, ptr, "Tokenizer", "Malformed number.")
end
local numStrFallback = numStr
if allowJitSyntax then
if s:find("^[Ii]", i2 + 1) then -- Imaginary part of complex number.
numStr = s:sub(i1, i2 + 1)
i2 = i2 + 1
elseif not maybeInt or numStr:find(".", 1, true) then
-- void
elseif s:find("^[Uu][Ll][Ll]", i2 + 1) then -- Unsigned 64-bit integer.
numStr = s:sub(i1, i2 + 3)
i2 = i2 + 3
elseif s:find("^[Ll][Ll]", i2 + 1) then -- Signed 64-bit integer.
numStr = s:sub(i1, i2 + 2)
i2 = i2 + 2
end
end
local n = tonumber(numStr) or tonumber(numStrFallback)
-- Support hexadecimal floats in Lua 5.1.
if not n and lua52Hex then
-- Note: We know we're not running LuaJIT here as it supports hexadecimal floats, thus we use numStrFallback instead of numStr.
local _, intStr, fracStr, expStr
if pat == NUM_HEX_FRAC_EXP then
_, intStr, fracStr, expStr = numStrFallback:match(NUM_HEX_FRAC_EXP)
elseif pat == NUM_HEX_FRAC then
_, intStr, fracStr = numStrFallback:match(NUM_HEX_FRAC)
expStr = "0"
elseif pat == NUM_HEX_EXP then
_, intStr, expStr = numStrFallback:match(NUM_HEX_EXP)
fracStr = ""
else
internalError()
end
n = tonumber(intStr, 16) or 0 -- intStr may be "".
local fracValue = 1
for i = 1, #fracStr do
fracValue = fracValue / 16
n = n + tonumber(fracStr:sub(i, i), 16) * fracValue
end
n = n * 2 ^ expStr:gsub("^+", "")
end
if not n then
errorInFile(s, path, ptr, "Tokenizer", "Invalid number.")
end
if s:find("^[%w_]", i2 + 1) then
-- This is actually not an error in Lua 5.2 and 5.3. Maybe we should issue a warning instead of an error here?
errorInFile(s, path, i2 + 1, "Tokenizer", "Malformed number.")
end
ptr = i2 + 1
tok = { type = "number", representation = numStrFallback, value = n }
-- Comment.
elseif s:find("^%-%-", ptr) then
local reprStart = ptr
ptr = ptr + 2
tok, ptr = parseStringlikeToken(s, ptr)
if not tok then
local errCode = ptr
if errCode == ERROR_UNFINISHED_STRINGLIKE then
errorInFile(s, path, reprStart, "Tokenizer", "Unfinished long comment.")
else
errorInFile(s, path, reprStart, "Tokenizer", "Invalid comment.")
end
end
if tok.long then
-- Check for nesting of [[...]], which is deprecated in Lua.
local chunk, err = loadLuaString("--" .. tok.representation, "@", nil)
if not chunk then
local lnInString, luaErr = err:match("^:(%d+): (.*)")
if luaErr then
errorOnLine(
path,
getLineNumber(s, reprStart) + tonumber(lnInString) - 1,
"Tokenizer",
"Malformed long comment. (%s)",
luaErr
)
else
errorInFile(s, path, reprStart, "Tokenizer", "Malformed long comment.")
end
end
end
tok.type = "comment"
tok.representation = s:sub(reprStart, ptr - 1)
-- String (short).
elseif s:find([=[^["']]=], ptr) then
local reprStart = ptr
local reprEnd
local quoteChar = s:sub(ptr, ptr)
ptr = ptr + 1
local valueStart = ptr
local valueEnd
while true do
local c = s:sub(ptr, ptr)
if c == "" then
errorInFile(s, path, reprStart, "Tokenizer", "Unfinished string.")
elseif c == quoteChar then
reprEnd = ptr
valueEnd = ptr - 1
ptr = reprEnd + 1
break
elseif c == "\\" then
-- Note: We don't have to look for multiple characters after
-- the escape, like \nnn - this algorithm works anyway.
if ptr + 1 > #s then
errorInFile(s, path, reprStart, "Tokenizer", "Unfinished string after escape.")
end
ptr = ptr + 2
elseif c == "\n" then
-- Can't have unescaped newlines. Lua, this is a silly rule! @Ugh
errorInFile(s, path, ptr, "Tokenizer", "Newlines must be escaped in strings.")
else
ptr = ptr + 1
end
end
local repr = s:sub(reprStart, reprEnd)
local valueChunk = loadLuaString("return" .. repr, nil, nil)
if not valueChunk then
errorInFile(s, path, reprStart, "Tokenizer", "Malformed string.")
end
local v = valueChunk()
assert(type(v) == "string")
tok = { type = "string", representation = repr, value = valueChunk(), long = false }
-- Long string.
elseif s:find("^%[=*%[", ptr) then
local reprStart = ptr
tok, ptr = parseStringlikeToken(s, ptr)
if not tok then
local errCode = ptr
if errCode == ERROR_UNFINISHED_STRINGLIKE then
errorInFile(s, path, reprStart, "Tokenizer", "Unfinished long string.")
else
errorInFile(s, path, reprStart, "Tokenizer", "Invalid long string.")
end
end
-- Check for nesting of [[...]], which is deprecated in Lua.
local valueChunk, err = loadLuaString("return" .. tok.representation, "@", nil)
if not valueChunk then
local lnInString, luaErr = err:match("^:(%d+): (.*)")
if luaErr then
errorOnLine(
path,
getLineNumber(s, reprStart) + tonumber(lnInString) - 1,
"Tokenizer",
"Malformed long string. (%s)",
luaErr
)
else
errorInFile(s, path, reprStart, "Tokenizer", "Malformed long string.")
end
end
local v = valueChunk()
assert(type(v) == "string")
tok.type = "string"
tok.value = v
-- Backtick string.
elseif s:find("^`", ptr) then
if not allowBacktickStrings then
errorInFile(s, path, ptr, "Tokenizer", "Encountered backtick string. (Feature not enabled.)")
end
local i1, i2, repr, v = s:find("^(`([^`]*)`)", ptr)
if not i2 then
errorInFile(s, path, ptr, "Tokenizer", "Unfinished backtick string.")
end
ptr = i2 + 1
tok = { type = "string", representation = repr, value = v, long = false }
-- Punctuation etc.
elseif s:find("^%.%.%.", ptr) then -- 3
local repr = s:sub(ptr, ptr + 2)
tok = { type = "punctuation", representation = repr, value = repr }
ptr = ptr + #repr
elseif
s:find("^%.%.", ptr)
or s:find("^[=~<>]=", ptr)
or s:find("^::", ptr)
or s:find("^//", ptr)
or s:find("^<<", ptr)
or s:find("^>>", ptr)
then -- 2
local repr = s:sub(ptr, ptr + 1)
tok = { type = "punctuation", representation = repr, value = repr }
ptr = ptr + #repr
elseif s:find("^[+%-*/%%^#<>=(){}[%];:,.&|~]", ptr) then -- 1
local repr = s:sub(ptr, ptr)
tok = { type = "punctuation", representation = repr, value = repr }
ptr = ptr + #repr
-- Preprocessor entry.
elseif s:find("^!", ptr) then
if not allowPpTokens then
errorInFile(s, path, ptr, "Tokenizer", "Encountered preprocessor entry. (Feature not enabled.)")
end
local double = s:find("^!", ptr + 1) ~= nil
local repr = s:sub(ptr, ptr + (double and 1 or 0))
tok = { type = "pp_entry", representation = repr, value = repr, double = double }
ptr = ptr + #repr
-- Preprocessor keyword.
elseif s:find("^@", ptr) then
if not allowPpTokens then
errorInFile(s, path, ptr, "Tokenizer", "Encountered preprocessor keyword. (Feature not enabled.)")
end
if s:find("^@@", ptr) then
ptr = ptr + 2
tok = { type = "pp_keyword", representation = "@@", value = "insert" }
else
local i1, i2, repr, word = s:find("^(@([%a_][%w_]*))", ptr)
if not i1 then
errorInFile(s, path, ptr + 1, "Tokenizer", "Expected an identifier.")
elseif not PREPROCESSOR_KEYWORDS[word] then
errorInFile(s, path, ptr + 1, "Tokenizer", "Invalid preprocessor keyword '%s'.", word)
end
ptr = i2 + 1
tok = { type = "pp_keyword", representation = repr, value = word }
end
-- Preprocessor symbol.
elseif s:find("^%$", ptr) then
if not allowPpTokens then
errorInFile(s, path, ptr, "Tokenizer", "Encountered preprocessor symbol. (Feature not enabled.)")
end
local i1, i2, repr, word = s:find("^(%$([%a_][%w_]*))", ptr)
if not i1 then
errorInFile(s, path, ptr + 1, "Tokenizer", "Expected an identifier.")
elseif KEYWORDS[word] then
errorInFile(
s,
path,
ptr + 1,
"Tokenizer",
"Invalid preprocessor symbol '%s'. (Must not be a Lua keyword.)",
word
)
end
ptr = i2 + 1
tok = { type = "pp_symbol", representation = repr, value = word }
else
errorInFile(s, path, ptr, "Tokenizer", "Unknown character.")
end
tok.line = ln
tok.position = tokenPos
tok.file = path
ln = ln + countString(tok.representation, "\n", true)
tok.lineEnd = ln
tableInsert(tokens, tok)
-- print(#tokens, tok.type, tok.representation) -- DEBUG
end
return tokens
end
-- luaString = _concatTokens( tokens, lastLn=nil, addLineNumbers, fromIndex=1, toIndex=#tokens )
local function _concatTokens(tokens, lastLn, addLineNumbers, i1, i2)
local parts = {}
if addLineNumbers then
for i = (i1 or 1), (i2 or #tokens) do
local tok = tokens[i]
lastLn = maybeOutputLineNumber(parts, tok, lastLn)
tableInsert(parts, tok.representation)
end
else
for i = (i1 or 1), (i2 or #tokens) do
tableInsert(parts, tokens[i].representation)
end
end
return table.concat(parts)
end
local function insertTokenRepresentations(parts, tokens, i1, i2)
for i = i1, i2 do
tableInsert(parts, tokens[i].representation)
end
end
local function readFile(path, isTextFile)
assertarg(1, path, "string")
assertarg(2, isTextFile, "boolean", "nil")
local file, err = io.open(path, "r" .. (isTextFile and "" or "b"))
if not file then
return nil, err
end
local contents = file:read("*a")
file:close()
return contents
end
-- success, error = writeFile( path, [ isTextFile=false, ] contents )
local function writeFile(path, isTextFile, contents)
assertarg(1, path, "string")
if type(isTextFile) == "boolean" then
assertarg(3, contents, "string")
else
isTextFile, contents = false, isTextFile
assertarg(2, contents, "string")
end
local file, err = io.open(path, "w" .. (isTextFile and "" or "b"))
if not file then
return false, err
end
file:write(contents)
file:close()
return true
end
local function fileExists(path)
assertarg(1, path, "string")
local file = io.open(path, "r")
if not file then
return false
end
file:close()
return true
end
-- assertarg( argumentNumber, value, expectedValueType1, ... )
--[[local]]
function assertarg(n, v, ...)
local vType = type(v)
for i = 1, select("#", ...) do
if vType == select(i, ...) then
return
end
end
local fName = debug.getinfo(2, "n").name
local expects = table.concat({ ... }, " or ")
if fName == "" then
fName = "?"
end
errorf(3, "bad argument #%d to '%s' (%s expected, got %s)", n, fName, expects, vType)
end
-- count = countString( haystack, needle [, plain=false ] )
--[[local]]
function countString(s, needle, plain)
local count = 0
local i = 0
local _
while true do
_, i = s:find(needle, i + 1, plain)
if not i then
return count
end
count = count + 1
end
end
-- count = countSubString( string, startPosition, endPosition, needle [, plain=false ] )
--[[local]]
function countSubString(s, pos, posEnd, needle, plain)
local count = 0
while true do
local _, i2 = s:find(needle, pos, plain)
if not i2 or i2 > posEnd then
return count
end
count = count + 1
pos = i2 + 1
end
end
local getfenv = getfenv
or function(f) -- Assume Lua is version 5.2+ if getfenv() doesn't exist.
f = f or 1
if type(f) == "function" then
-- void
elseif type(f) == "number" then
if f == 0 then
return _ENV
end
if f < 0 then
error("bad argument #1 to 'getfenv' (level must be non-negative)")
end
f = debug.getinfo(1 + f, "f") or error("bad argument #1 to 'getfenv' (invalid level)")
f = f.func
else
error("bad argument #1 to 'getfenv' (number expected, got " .. type(f) .. ")")
end
for i = 1, 1 / 0 do
local name, v = debug.getupvalue(f, i)
if name == "_ENV" then
return v
end
if not name then
return _ENV
end
end
end
-- (Table generated by misc/generateStringEscapeSequenceInfo.lua)
local UNICODE_RANGES_NOT_TO_ESCAPE = {
{ from = 32, to = 126 },
{ from = 161, to = 591 },
{ from = 880, to = 887 },
{ from = 890, to = 895 },
{ from = 900, to = 906 },
{ from = 908, to = 908 },
{ from = 910, to = 929 },
{ from = 931, to = 1154 },
{ from = 1162, to = 1279 },
{ from = 7682, to = 7683 },
{ from = 7690, to = 7691 },
{ from = 7710, to = 7711 },
{ from = 7744, to = 7745 },
{ from = 7766, to = 7767 },
{ from = 7776, to = 7777 },
{ from = 7786, to = 7787 },
{ from = 7808, to = 7813 },
{ from = 7835, to = 7835 },
{ from = 7922, to = 7923 },
{ from = 8208, to = 8208 },
{ from = 8210, to = 8231 },
{ from = 8240, to = 8286 },
{ from = 8304, to = 8305 },
{ from = 8308, to = 8334 },
{ from = 8336, to = 8348 },
{ from = 8352, to = 8383 },
{ from = 8448, to = 8587 },
{ from = 8592, to = 9254 },
{ from = 9312, to = 10239 },
{ from = 10496, to = 11007 },
{ from = 64256, to = 64262 },
}
local function shouldCodepointBeEscaped(cp)
for _, range in ipairs(UNICODE_RANGES_NOT_TO_ESCAPE) do -- @Speed: Don't use a loop?
if cp >= range.from and cp <= range.to then
return false
end
end
return true
end
-- local cache = setmetatable({}, {__mode="kv"}) -- :SerializationCache (This doesn't seem to speed things up.)
-- success, error = serialize( buffer, value )
local function serialize(buffer, v)
--[[ :SerializationCache
if cache[v] then
tableInsert(buffer, cache[v])
return true
end
local bufferStart = #buffer + 1
--]]
local vType = type(v)
if vType == "table" then
local first = true
tableInsert(buffer, "{")
local indices = {}
for i, item in ipairs(v) do
if not first then
tableInsert(buffer, ",")
end
first = false
local ok, err = serialize(buffer, item)
if not ok then
return false, err
end
indices[i] = true
end
local keys = {}
for k, item in pairs(v) do
if indices[k] then
-- void
elseif type(k) == "table" then
return false, "Table keys cannot be tables."
else
tableInsert(keys, k)
end
end
table.sort(keys, function(a, b)
return tostring(a) < tostring(b)
end)
for _, k in ipairs(keys) do
local item = v[k]
if not first then
tableInsert(buffer, ",")
end
first = false
if not KEYWORDS[k] and type(k) == "string" and k:find("^[%a_][%w_]*$") then
tableInsert(buffer, k)
tableInsert(buffer, "=")
else
tableInsert(buffer, "[")
local ok, err = serialize(buffer, k)
if not ok then
return false, err
end
tableInsert(buffer, "]=")
end
local ok, err = serialize(buffer, item)
if not ok then
return false, err
end
end
tableInsert(buffer, "}")
elseif vType == "string" then
if v == "" then
tableInsert(buffer, '""')
return true
end
local useApostrophe = v:find('"', 1, true) and not v:find("'", 1, true)
local quote = useApostrophe and "'" or '"'
tableInsert(buffer, quote)
if current_anytime_fastStrings or not v:find("[^\32-\126\t\n]") then
-- print(">> FAST", #v) -- DEBUG
local s = v:gsub((useApostrophe and "[\t\n\\']" or '[\t\n\\"]'), function(c)
return ESCAPE_SEQUENCES[c] or internalError(c:byte())
end)
tableInsert(buffer, s)
else
-- print(">> SLOW", #v) -- DEBUG
local pos = 1
-- @Speed: There are optimizations to be made here!
while pos <= #v do
local c = v:sub(pos, pos)
local cp, len = utf8GetCodepointAndLength(v, pos)
-- Named escape sequences.
if ESCAPE_SEQUENCES_EXCEPT_QUOTES[c] then
tableInsert(buffer, ESCAPE_SEQUENCES_EXCEPT_QUOTES[c])
pos = pos + 1
elseif c == quote then
tableInsert(buffer, [[\]])
tableInsert(buffer, quote)
pos = pos + 1
-- UTF-8 character.
elseif len == 1 and not shouldCodepointBeEscaped(cp) then
tableInsert(buffer, v:sub(pos, pos))
pos = pos + 1 -- @Speed: We can insert multiple single-byte characters sometimes!
elseif len and not shouldCodepointBeEscaped(cp) then
tableInsert(buffer, v:sub(pos, pos + len - 1))
pos = pos + len
-- Anything else.
else
tableInsert(buffer, F((v:find("^%d", pos + 1) and "\\%03d" or "\\%d"), v:byte(pos)))
pos = pos + 1
end
end
end
tableInsert(buffer, quote)
elseif v == 1 / 0 then
tableInsert(buffer, "(1/0)")
elseif v == -1 / 0 then
tableInsert(buffer, "(-1/0)")
elseif v ~= v then
tableInsert(buffer, "(0/0)") -- NaN.
elseif v == 0 then
tableInsert(buffer, "0") -- In case it's actually -0 for some reason, which would be silly to output.
elseif vType == "number" then
if v < 0 then
tableInsert(buffer, " ") -- The space prevents an accidental comment if a "-" is right before.
end
tableInsert(buffer, tostring(v)) -- (I'm not sure what precision tostring() uses for numbers. Maybe we should use string.format() instead.)
elseif vType == "boolean" or v == nil then
tableInsert(buffer, tostring(v))
else
return false, F("Cannot serialize value of type '%s'. (%s)", vType, tostring(v))
end
--[[ :SerializationCache
if v ~= nil then
cache[v] = table.concat(buffer, "", bufferStart, #buffer)
end
--]]
return true
end
-- luaString = toLua( value )
-- Returns nil and a message on error.
local function toLua(v)
local buffer = {}
local ok, err = serialize(buffer, v)
if not ok then
return nil, err
end
return table.concat(buffer)
end
-- value = evaluate( expression [, environment=getfenv() ] )
-- Returns nil and a message on error.
local function evaluate(expr, env)
local chunk, err = loadLuaString("return(" .. expr .. "\n)", "@<evaluate>", (env or getfenv(2)))
if not chunk then
return nil, F("Invalid expression '%s'. (%s)", expr, (err:gsub("^:%d+: ", "")))
end
local ok, valueOrErr = pcall(chunk)
if not ok then
return nil, valueOrErr
end
return valueOrErr -- May be nil or false!
end
local function escapePattern(s)
return (s:gsub("[-+*^?$.%%()[%]]", "%%%0"))
end
local function outputLineNumber(parts, ln)
tableInsert(parts, "--[[@")
tableInsert(parts, ln)
tableInsert(parts, "]]")
end
--[[local]]
function maybeOutputLineNumber(parts, tok, lastLn)
if tok.line == lastLn or USELESS_TOKENS[tok.type] then
return lastLn
end
outputLineNumber(parts, tok.line)
return tok.line
end
--[=[
--[[local]] function maybeOutputLineNumber(parts, tok, lastLn, fromMetaToOutput)
if tok.line == lastLn or USELESS_TOKENS[tok.type] then return lastLn end
if fromMetaToOutput then
tableInsert(parts, '__LUA"--[[@'..tok.line..']]"\n')
else
tableInsert(parts, "--[[@"..tok.line.."]]")
end
return tok.line
end
]=]
local function isAny(v, ...)
for i = 1, select("#", ...) do
if v == select(i, ...) then
return true
end
end
return false
end
local function errorIfNotRunningMeta(level)
if not current_anytime_isRunningMeta then
error("No file is being processed.", 1 + level)
end
end
local function copyArray(t)
local copy = {}
for i, v in ipairs(t) do
copy[i] = v
end
return copy
end
local copyTable
do
local function deepCopy(t, copy, tableCopies)
for k, v in pairs(t) do
if type(v) == "table" then
local vCopy = tableCopies[v]
if vCopy then
copy[k] = vCopy
else
vCopy = {}
tableCopies[v] = vCopy
copy[k] = deepCopy(v, vCopy, tableCopies)
end
else
copy[k] = v
end
end
return copy
end
-- copy = copyTable( table [, deep=false ] )
--[[local]]
function copyTable(t, deep)
local copy = {}
if deep then
return deepCopy(t, copy, { [t] = copy })
end
for k, v in pairs(t) do
copy[k] = v
end
return copy
end
end
-- values = pack( value1, ... )
-- values.n is the amount of values (which can be zero).
local pack = (
(_VERSION >= "Lua 5.2" or jit) and table.pack or function(...)
return { n = select("#", ...), ... }
end
)
local unpack = (_VERSION >= "Lua 5.2") and table.unpack or _G.unpack
--[[local]]
loadLuaString = (
(_VERSION >= "Lua 5.2" or jit)
and function(lua, chunkName, env)
return load(lua, chunkName, "bt", env)
end
or function(lua, chunkName, env)
local chunk, err = loadstring(lua, chunkName)
if not chunk then
return nil, err
end
if env then
setfenv(chunk, env)
end
return chunk
end
)
local loadLuaFile = (
(_VERSION >= "Lua 5.2" or jit) and function(path, env)
return loadfile(path, "bt", env)
end
or function(path, env)
local chunk, err = loadfile(path)
if not chunk then
return nil, err
end
if env then
setfenv(chunk, env)
end
return chunk
end
)
local function isLuaStringValidExpression(lua)
return loadLuaString("return(" .. lua .. "\n)", "@", nil) ~= nil
end
-- token, index = getNextUsableToken( tokens, startIndex, indexLimit=autoDependingOnDirection, direction )
local function getNextUsableToken(tokens, iStart, iLimit, dir)
iLimit = (dir < 0 and math.max((iLimit or 1), 1) or math.min((iLimit or 1 / 0), #tokens))
for i = iStart, iLimit, dir do
if not USELESS_TOKENS[tokens[i].type] then
return tokens[i], i
end
end
return nil
end
-- bool = isToken( token, tokenType [, tokenValue=any ] )
local function isToken(tok, tokType, v)
return tok.type == tokType and (v == nil or tok.value == v)
end
-- bool = isTokenAndNotNil( token, tokenType [, tokenValue=any ] )
local function isTokenAndNotNil(tok, tokType, v)
return tok ~= nil and tok.type == tokType and (v == nil or tok.value == v)
end
--[[local]]
function getLineNumber(s, pos)
return 1 + countSubString(s, 1, pos - 1, "\n", true)
end
-- text = getRelativeLocationText( tokenOfInterest, otherToken )
-- text = getRelativeLocationText( tokenOfInterest, otherFilename, otherLineNumber )
local function getRelativeLocationText(tokOfInterest, otherFilename, otherLn)
if type(otherFilename) == "table" then
return getRelativeLocationText(tokOfInterest, otherFilename.file, otherFilename.line)
end
if not (tokOfInterest.file and tokOfInterest.line) then
return "at <UnknownLocation>"
end
if tokOfInterest.file ~= otherFilename then
return F("at %s:%d", tokOfInterest.file, tokOfInterest.line)
end
if tokOfInterest.line + 1 == otherLn then
return F("on the previous line")
end
if tokOfInterest.line - 1 == otherLn then
return F("on the next line")
end
if tokOfInterest.line ~= otherLn then
return F("on line %d", tokOfInterest.line)
end
return "on the same line"
end
--[[local]]
tableInsert = table.insert
--[[local]]
tableRemove = table.remove
--[[local]]
function tableInsertFormat(t, s, ...)
tableInsert(t, F(s, ...))
end
-- length|nil = utf8GetCharLength( string [, position=1 ] )
local function utf8GetCharLength(s, pos)
pos = pos or 1
local b1, b2, b3, b4 = s:byte(pos, pos + 3)
if b1 > 0 and b1 <= 127 then
return 1
elseif b1 >= 194 and b1 <= 223 then
if not b2 then
return nil
end -- UTF-8 string terminated early.
if b2 < 128 or b2 > 191 then
return nil
end -- Invalid UTF-8 character.
return 2
elseif b1 >= 224 and b1 <= 239 then
if not b3 then
return nil
end -- UTF-8 string terminated early.
if b1 == 224 and (b2 < 160 or b2 > 191) then
return nil
end -- Invalid UTF-8 character.
if b1 == 237 and (b2 < 128 or b2 > 159) then
return nil
end -- Invalid UTF-8 character.
if b2 < 128 or b2 > 191 then
return nil
end -- Invalid UTF-8 character.
if b3 < 128 or b3 > 191 then
return nil
end -- Invalid UTF-8 character.
return 3
elseif b1 >= 240 and b1 <= 244 then
if not b4 then
return nil
end -- UTF-8 string terminated early.
if b1 == 240 and (b2 < 144 or b2 > 191) then
return nil
end -- Invalid UTF-8 character.
if b1 == 244 and (b2 < 128 or b2 > 143) then
return nil
end -- Invalid UTF-8 character.
if b2 < 128 or b2 > 191 then
return nil
end -- Invalid UTF-8 character.
if b3 < 128 or b3 > 191 then
return nil
end -- Invalid UTF-8 character.
if b4 < 128 or b4 > 191 then
return nil
end -- Invalid UTF-8 character.
return 4
end
return nil -- Invalid UTF-8 character.
end
-- codepoint, length = utf8GetCodepointAndLength( string [, position=1 ] )
-- Returns nil if the text is invalid at the position.
--[[local]]
function utf8GetCodepointAndLength(s, pos)
pos = pos or 1
local len = utf8GetCharLength(s, pos)
if not len then
return nil
end
-- 2^6=64, 2^12=4096, 2^18=262144
if len == 1 then
return s:byte(pos), len
end
if len == 2 then
local b1, b2 = s:byte(pos, pos + 1)
return (b1 - 192) * 64 + (b2 - 128), len
end
if len == 3 then
local b1, b2, b3 = s:byte(pos, pos + 2)
return (b1 - 224) * 4096 + (b2 - 128) * 64 + (b3 - 128), len
end
do
local b1, b2, b3, b4 = s:byte(pos, pos + 3)
return (b1 - 240) * 262144 + (b2 - 128) * 4096 + (b3 - 128) * 64 + (b4 - 128), len
end
end
-- for k, v in pairsSorted( table ) do
local function pairsSorted(t)
local keys = {}
for k in pairs(t) do
tableInsert(keys, k)
end
sortNatural(keys)
local i = 0
return function()
i = i + 1
local k = keys[i]
if k ~= nil then
return k, t[k]
end
end
end
-- sortNatural( array )
-- aIsLessThanB = compareNatural( a, b )
local compareNatural
do
local function pad(numStr)
return F("%03d%s", #numStr, numStr)
end
--[[local]]
function compareNatural(a, b)
if type(a) == "number" and type(b) == "number" then
return a < b
else
return (tostring(a):gsub("%d+", pad) < tostring(b):gsub("%d+", pad))
end
end
--[[local]]
function sortNatural(t, k)
table.sort(t, compareNatural)
end
end
-- lua = _loadResource( resourceName, isParsing==true , nameToken, stats ) -- At parse time.
-- lua = _loadResource( resourceName, isParsing==false, errorLevel ) -- At metaprogram runtime.
local function _loadResource(resourceName, isParsing, nameTokOrErrLevel, stats)
local lua = current_parsingAndMeta_resourceCache[resourceName]
if not lua then
if current_parsingAndMeta_onInsert then
lua = current_parsingAndMeta_onInsert(resourceName)
if type(lua) == "string" then
-- void
elseif isParsing then
errorAtToken(
nameTokOrErrLevel,
nameTokOrErrLevel.position + 1,
"Parser/MetaProgram",
"Expected a string from params.onInsert(). (Got %s)",
type(lua)
)
else
errorf(1 + nameTokOrErrLevel, "Expected a string from params.onInsert(). (Got %s)", type(lua))
end
else
local err
lua, err = readFile(resourceName, true)
if lua then
-- void
elseif isParsing then
errorAtToken(
nameTokOrErrLevel,
nameTokOrErrLevel.position + 1,
"Parser",
"Could not read file '%s'. (%s)",
resourceName,
tostring(err)
)
else
errorf(1 + nameTokOrErrLevel, "Could not read file '%s'. (%s)", resourceName, tostring(err))
end
end
current_parsingAndMeta_resourceCache[resourceName] = lua
if isParsing then
tableInsert(stats.insertedNames, resourceName)
end
elseif isParsing then
current_parsing_insertCount = current_parsing_insertCount + 1 -- Note: We don't count insertions of newly encountered files.
if current_parsing_insertCount > MAX_DUPLICATE_FILE_INSERTS then
errorAtToken(
nameTokOrErrLevel,
nameTokOrErrLevel.position + 1,
"Parser",
"Too many duplicate inserts. We may be stuck in a recursive loop. (Unique files inserted so far: %s)",
stats.insertedNames[1] and table.concat(stats.insertedNames, ", ") or "none"
)
end
end
return lua
end
--==============================================================
--= Preprocessor Functions =====================================
--==============================================================
-- :EnvironmentTable
----------------------------------------------------------------
metaEnv = copyTable(_G) -- Include all standard Lua stuff.
metaEnv._G = metaEnv
local metaFuncs = {}
-- printf()
-- printf( format, value1, ... )
-- Print a formatted string to stdout.
metaFuncs.printf = printf
-- readFile()
-- contents = readFile( path [, isTextFile=false ] )
-- Get the entire contents of a binary file or text file. Returns nil and a message on error.
metaFuncs.readFile = readFile
metaFuncs.getFileContents = readFile -- @Deprecated
-- writeFile()
-- success, error = writeFile( path, contents ) -- Writes a binary file.
-- success, error = writeFile( path, isTextFile, contents )
-- Write an entire binary file or text file.
metaFuncs.writeFile = writeFile
-- fileExists()
-- bool = fileExists( path )
-- Check if a file exists.
metaFuncs.fileExists = fileExists
-- toLua()
-- luaString = toLua( value )
-- Convert a value to a Lua literal. Does not work with certain types, like functions or userdata.
-- Returns nil and a message on error.
metaFuncs.toLua = toLua
-- serialize()
-- success, error = serialize( buffer, value )
-- Same as toLua() except adds the result to an array instead of returning the Lua code as a string.
-- This could avoid allocating unnecessary strings.
metaFuncs.serialize = serialize
-- evaluate()
-- value = evaluate( expression [, environment=getfenv() ] )
-- Evaluate a Lua expression. The function is kind of the opposite of toLua(). Returns nil and a message on error.
-- Note that nil or false can also be returned as the first value if that's the value the expression results in!
metaFuncs.evaluate = evaluate
-- escapePattern()
-- escapedString = escapePattern( string )
-- Escape a string so it can be used in a pattern as plain text.
metaFuncs.escapePattern = escapePattern
-- isToken()
-- bool = isToken( token, tokenType [, tokenValue=any ] )
-- Check if a token is of a specific type, optionally also check it's value.
metaFuncs.isToken = isToken
-- copyTable()
-- copy = copyTable( table [, deep=false ] )
-- Copy a table, optionally recursively (deep copy).
-- Multiple references to the same table and self-references are preserved during deep copying.
metaFuncs.copyTable = copyTable
-- unpack()
-- value1, ... = unpack( array [, fromIndex=1, toIndex=#array ] )
-- Is _G.unpack() in Lua 5.1 and alias for table.unpack() in Lua 5.2+.
metaFuncs.unpack = unpack
-- pack()
-- values = pack( value1, ... )
-- Put values in a new array. values.n is the amount of values (which can be zero)
-- including nil values. Alias for table.pack() in Lua 5.2+.
metaFuncs.pack = pack
-- pairsSorted()
-- for key, value in pairsSorted( table ) do
-- Same as pairs() but the keys are sorted (ascending).
metaFuncs.pairsSorted = pairsSorted
-- sortNatural()
-- sortNatural( array )
-- Sort an array using compareNatural().
metaFuncs.sortNatural = sortNatural
-- compareNatural()
-- aIsLessThanB = compareNatural( a, b )
-- Compare two strings. Numbers in the strings are compared as numbers (as opposed to as strings).
-- Examples:
-- print( "foo9" < "foo10" ) -- false
-- print(compareNatural("foo9", "foo10")) -- true
metaFuncs.compareNatural = compareNatural
-- run()
-- returnValue1, ... = run( path [, arg1, ... ] )
-- Execute a Lua file. Similar to dofile().
function metaFuncs.run(path, ...)
assertarg(1, path, "string")
local main_chunk, err = loadLuaFile(path, metaEnv)
if not main_chunk then
error(err, 0)
end
-- We want multiple return values while avoiding a tail call to preserve stack info.
local returnValues = pack(main_chunk(...))
return unpack(returnValues, 1, returnValues.n)
end
-- outputValue()
-- outputValue( value )
-- outputValue( value1, value2, ... ) -- Outputted values will be separated by commas.
-- Output one or more values, like strings or tables, as literals.
-- Raises an error if no file or string is being processed.
function metaFuncs.outputValue(...)
errorIfNotRunningMeta(2)
local argCount = select("#", ...)
if argCount == 0 then
error("No values to output.", 2)
end
for i = 1, argCount do
local v = select(i, ...)
if v == nil and not current_meta_canOutputNil then
local ln = debug.getinfo(2, "l").currentline
errorOnLine(
current_meta_pathForErrorMessages,
ln,
"MetaProgram",
"Trying to output nil which is disallowed through params.canOutputNil ."
)
end
if i > 1 then
tableInsert(current_meta_output, (current_parsingAndMeta_isDebug and ", " or ","))
end
local ok, err = serialize(current_meta_output, v)
if not ok then
local ln = debug.getinfo(2, "l").currentline
errorOnLine(current_meta_pathForErrorMessages, ln, "MetaProgram", "%s", err)
end
end
end
-- outputLua()
-- outputLua( luaString1, ... )
-- Output one or more strings as raw Lua code.
-- Raises an error if no file or string is being processed.
function metaFuncs.outputLua(...)
errorIfNotRunningMeta(2)
local argCount = select("#", ...)
if argCount == 0 then
error("No Lua code to output.", 2)
end
for i = 1, argCount do
local lua = select(i, ...)
assertarg(i, lua, "string")
tableInsert(current_meta_output, lua)
end
end
-- outputLuaTemplate()
-- outputLuaTemplate( luaStringTemplate, value1, ... )
-- Use a string as a template for outputting Lua code with values.
-- Question marks (?) are replaced with the values.
-- Raises an error if no file or string is being processed.
-- Examples:
-- outputLuaTemplate("local name, age = ?, ?", "Harry", 48)
-- outputLuaTemplate("dogs[?] = ?", "greyhound", {italian=false, count=5})
function metaFuncs.outputLuaTemplate(lua, ...)
errorIfNotRunningMeta(2)
assertarg(1, lua, "string")
local args = { ... } -- @Memory
local n = 0
local v, err
lua = lua:gsub("%?", function()
n = n + 1
v, err = toLua(args[n])
if not v then
errorf(3, "Bad argument %d: %s", 1 + n, err)
end
return v
end)
tableInsert(current_meta_output, lua)
end
-- getOutputSoFar()
-- luaString = getOutputSoFar( [ asTable=false ] )
-- getOutputSoFar( buffer )
-- Get Lua code that's been outputted so far.
-- If asTable is false then the full Lua code string is returned.
-- If asTable is true then an array of Lua code segments is returned. (This avoids allocating, possibly large, strings.)
-- If a buffer array is given then Lua code segments are added to it.
-- Raises an error if no file or string is being processed.
function metaFuncs.getOutputSoFar(bufferOrAsTable)
errorIfNotRunningMeta(2)
-- Should there be a way to get the contents of current_meta_output etc.? :GetMoreOutputFromStack
if type(bufferOrAsTable) == "table" then
for _, lua in ipairs(current_meta_outputStack[1]) do
tableInsert(bufferOrAsTable, lua)
end
-- Return nothing!
else
return bufferOrAsTable and copyArray(current_meta_outputStack[1]) or table.concat(current_meta_outputStack[1])
end
end
local lineFragments = {}
local function getOutputSoFarOnLine()
errorIfNotRunningMeta(2)
local len = 0
-- Should there be a way to get the contents of current_meta_output etc.? :GetMoreOutputFromStack
for i = #current_meta_outputStack[1], 1, -1 do
local fragment = current_meta_outputStack[1][i]
if fragment:find("\n", 1, true) then
len = len + 1
lineFragments[len] = fragment:gsub(".*\n", "")
break
end
len = len + 1
lineFragments[len] = fragment
end
return table.concat(lineFragments, 1, len)
end
-- getOutputSoFarOnLine()
-- luaString = getOutputSoFarOnLine( )
-- Get Lua code that's been outputted so far on the current line.
-- Raises an error if no file or string is being processed.
metaFuncs.getOutputSoFarOnLine = getOutputSoFarOnLine
-- getOutputSizeSoFar()
-- size = getOutputSizeSoFar( )
-- Get the amount of bytes outputted so far.
-- Raises an error if no file or string is being processed.
function metaFuncs.getOutputSizeSoFar()
errorIfNotRunningMeta(2)
local size = 0
for _, lua in ipairs(current_meta_outputStack[1]) do -- :GetMoreOutputFromStack
size = size + #lua
end
return size
end
-- getCurrentLineNumberInOutput()
-- lineNumber = getCurrentLineNumberInOutput( )
-- Get the current line number in the output.
function metaFuncs.getCurrentLineNumberInOutput()
errorIfNotRunningMeta(2)
local ln = 1
for _, lua in ipairs(current_meta_outputStack[1]) do -- :GetMoreOutputFromStack
ln = ln + countString(lua, "\n", true)
end
return ln
end
local function getIndentation(line, tabWidth)
if not tabWidth then
return line:match("^[ \t]*")
end
local indent = 0
for i = 1, #line do
if line:sub(i, i) == "\t" then
indent = math.floor(indent / tabWidth) * tabWidth + tabWidth
elseif line:sub(i, i) == " " then
indent = indent + 1
else
break
end
end
return indent
end
-- getIndentation()
-- string = getIndentation( line )
-- size = getIndentation( line, tabWidth )
-- Get indentation of a line, either as a string or as a size in spaces.
metaFuncs.getIndentation = getIndentation
-- getCurrentIndentationInOutput()
-- string = getCurrentIndentationInOutput( )
-- size = getCurrentIndentationInOutput( tabWidth )
-- Get the indentation of the current line, either as a string or as a size in spaces.
function metaFuncs.getCurrentIndentationInOutput(tabWidth)
errorIfNotRunningMeta(2)
return (getIndentation(getOutputSoFarOnLine(), tabWidth))
end
-- getCurrentPathIn()
-- path = getCurrentPathIn( )
-- Get what file is currently being processed, if any.
function metaFuncs.getCurrentPathIn()
return current_anytime_pathIn
end
-- getCurrentPathOut()
-- path = getCurrentPathOut( )
-- Get what file the currently processed file will be written to, if any.
function metaFuncs.getCurrentPathOut()
return current_anytime_pathOut
end
-- tokenize()
-- tokens = tokenize( luaString [, allowPreprocessorCode=false ] )
-- token = {
-- type=tokenType, representation=representation, value=value,
-- line=lineNumber, lineEnd=lineNumber, position=bytePosition, file=filePath,
-- ...
-- }
-- Convert Lua code to tokens. Returns nil and a message on error. (See newToken() for token types.)
function metaFuncs.tokenize(lua, allowPpCode)
local ok, errOrTokens = pcall(_tokenize, lua, "<string>", allowPpCode, allowPpCode, true) -- @Incomplete: Make allowJitSyntax a parameter to tokenize()?
if not ok then
return nil, cleanError(errOrTokens)
end
return errOrTokens
end
-- removeUselessTokens()
-- removeUselessTokens( tokens )
-- Remove whitespace and comment tokens.
function metaFuncs.removeUselessTokens(tokens)
local len = #tokens
local offset = 0
for i, tok in ipairs(tokens) do
if USELESS_TOKENS[tok.type] then
offset = offset - 1
else
tokens[i + offset] = tok
end
end
for i = len, len + offset + 1, -1 do
tokens[i] = nil
end
end
local function nextUsefulToken(tokens, i)
while true do
i = i + 1
local tok = tokens[i]
if not tok then
return
end
if not USELESS_TOKENS[tok.type] then
return i, tok
end
end
end
-- eachToken()
-- for index, token in eachToken( tokens [, ignoreUselessTokens=false ] ) do
-- Loop through tokens.
function metaFuncs.eachToken(tokens, ignoreUselessTokens)
if ignoreUselessTokens then
return nextUsefulToken, tokens, 0
else
return ipairs(tokens)
end
end
-- getNextUsefulToken()
-- token, index = getNextUsefulToken( tokens, startIndex [, steps=1 ] )
-- Get the next token that isn't a whitespace or comment. Returns nil if no more tokens are found.
-- Specify a negative steps value to get an earlier token.
function metaFuncs.getNextUsefulToken(tokens, i1, steps)
steps = (steps or 1)
local i2, dir
if steps == 0 then
return tokens[i1], i1
elseif steps < 0 then
i2, dir = 1, -1
else
i2, dir = #tokens, 1
end
for i = i1, i2, dir do
local tok = tokens[i]
if not USELESS_TOKENS[tok.type] then
steps = steps - dir
if steps == 0 then
return tok, i
end
end
end
return nil
end
local numberFormatters = {
auto = function(n)
return tostring(n)
end,
integer = function(n)
return F("%d", n)
end,
int = function(n)
return F("%d", n)
end,
float = function(n)
return F("%f", n):gsub("(%d)0+$", "%1")
end,
scientific = function(n)
return F("%e", n):gsub("(%d)0+e", "%1e"):gsub("0+(%d+)$", "%1")
end,
SCIENTIFIC = function(n)
return F("%E", n):gsub("(%d)0+E", "%1E"):gsub("0+(%d+)$", "%1")
end,
e = function(n)
return F("%e", n):gsub("(%d)0+e", "%1e"):gsub("0+(%d+)$", "%1")
end,
E = function(n)
return F("%E", n):gsub("(%d)0+E", "%1E"):gsub("0+(%d+)$", "%1")
end,
hexadecimal = function(n)
return (n == math.floor(n) and F("0x%x", n) or error("Hexadecimal floats not supported yet.", 3))
end, -- @Incomplete
HEXADECIMAL = function(n)
return (n == math.floor(n) and F("0x%X", n) or error("Hexadecimal floats not supported yet.", 3))
end,
hex = function(n)
return (n == math.floor(n) and F("0x%x", n) or error("Hexadecimal floats not supported yet.", 3))
end,
HEX = function(n)
return (n == math.floor(n) and F("0x%X", n) or error("Hexadecimal floats not supported yet.", 3))
end,
}
-- newToken()
-- token = newToken( tokenType, ... )
-- Create a new token. Different token types take different arguments.
--
-- commentToken = newToken( "comment", contents [, forceLongForm=false ] )
-- identifierToken = newToken( "identifier", identifier )
-- keywordToken = newToken( "keyword", keyword )
-- numberToken = newToken( "number", number [, numberFormat="auto" ] )
-- punctuationToken = newToken( "punctuation", symbol )
-- stringToken = newToken( "string", contents [, longForm=false ] )
-- whitespaceToken = newToken( "whitespace", contents )
-- ppEntryToken = newToken( "pp_entry", isDouble )
-- ppKeywordToken = newToken( "pp_keyword", ppKeyword ) -- ppKeyword can be "file", "insert", "line" or "@".
-- ppSymbolToken = newToken( "pp_symbol", identifier )
--
-- commentToken = { type="comment", representation=string, value=string, long=isLongForm }
-- identifierToken = { type="identifier", representation=string, value=string }
-- keywordToken = { type="keyword", representation=string, value=string }
-- numberToken = { type="number", representation=string, value=number }
-- punctuationToken = { type="punctuation", representation=string, value=string }
-- stringToken = { type="string", representation=string, value=string, long=isLongForm }
-- whitespaceToken = { type="whitespace", representation=string, value=string }
-- ppEntryToken = { type="pp_entry", representation=string, value=string, double=isDouble }
-- ppKeywordToken = { type="pp_keyword", representation=string, value=string }
-- ppSymbolToken = { type="pp_symbol", representation=string, value=string }
--
-- Number formats:
-- "integer" E.g. 42
-- "int" Same as integer, e.g. 42
-- "float" E.g. 3.14
-- "scientific" E.g. 0.7e+12
-- "SCIENTIFIC" E.g. 0.7E+12 (upper case)
-- "e" Same as scientific, e.g. 0.7e+12
-- "E" Same as SCIENTIFIC, e.g. 0.7E+12 (upper case)
-- "hexadecimal" E.g. 0x19af
-- "HEXADECIMAL" E.g. 0x19AF (upper case)
-- "hex" Same as hexadecimal, e.g. 0x19af
-- "HEX" Same as HEXADECIMAL, e.g. 0x19AF (upper case)
-- "auto" Note: Infinite numbers and NaN always get automatic format.
--
function metaFuncs.newToken(tokType, ...)
if tokType == "comment" then
local comment, long = ...
long = not not (long or comment:find("[\r\n]"))
assertarg(2, comment, "string")
local repr
if long then
local equalSigns = ""
while comment:find(F("]%s]", equalSigns), 1, true) do
equalSigns = equalSigns .. "="
end
repr = F("--[%s[%s]%s]", equalSigns, comment, equalSigns)
else
repr = F("--%s\n", comment)
end
return { type = "comment", representation = repr, value = comment, long = long }
elseif tokType == "identifier" then
local ident = ...
assertarg(2, ident, "string")
if ident == "" then
error("Identifier length is 0.", 2)
elseif not ident:find("^[%a_][%w_]*$") then
errorf(2, "Bad identifier format: '%s'", ident)
elseif KEYWORDS[ident] then
errorf(2, "Identifier must not be a keyword: '%s'", ident)
end
return { type = "identifier", representation = ident, value = ident }
elseif tokType == "keyword" then
local keyword = ...
assertarg(2, keyword, "string")
if not KEYWORDS[keyword] then
errorf(2, "Bad keyword '%s'.", keyword)
end
return { type = "keyword", representation = keyword, value = keyword }
elseif tokType == "number" then
local n, numberFormat = ...
numberFormat = numberFormat or "auto"
assertarg(2, n, "number")
assertarg(3, numberFormat, "string")
-- Some of these are technically multiple other tokens. We could raise an error but ehhh...
local numStr = (
n ~= n and "(0/0)"
or n == 1 / 0 and "(1/0)"
or n == -1 / 0 and "(-1/0)"
or numberFormatters[numberFormat] and numberFormatters[numberFormat](n)
or errorf(2, "Invalid number format '%s'.", numberFormat)
)
return { type = "number", representation = numStr, value = n }
elseif tokType == "punctuation" then
local symbol = ...
assertarg(2, symbol, "string")
-- Note: "!" and "!!" are of a different token type (pp_entry).
if not PUNCTUATION[symbol] then
errorf(2, "Bad symbol '%s'.", symbol)
end
return { type = "punctuation", representation = symbol, value = symbol }
elseif tokType == "string" then
local s, long = ...
long = not not long
assertarg(2, s, "string")
local repr
if long then
local equalSigns = ""
while s:find(F("]%s]", equalSigns), 1, true) do
equalSigns = equalSigns .. "="
end
repr = F("[%s[%s]%s]", equalSigns, s, equalSigns)
else
repr = toLua(s)
end
return { type = "string", representation = repr, value = s, long = long }
elseif tokType == "whitespace" then
local whitespace = ...
assertarg(2, whitespace, "string")
if whitespace == "" then
error("String is empty.", 2)
elseif whitespace:find("%S") then
error("String contains non-whitespace characters.", 2)
end
return { type = "whitespace", representation = whitespace, value = whitespace }
elseif tokType == "pp_entry" then
local double = ...
assertarg(2, double, "boolean")
local symbol = double and "!!" or "!"
return { type = "pp_entry", representation = symbol, value = symbol, double = double }
elseif tokType == "pp_keyword" then
local keyword = ...
assertarg(2, keyword, "string")
if keyword == "@" then
return { type = "pp_keyword", representation = "@@", value = "insert" }
elseif not PREPROCESSOR_KEYWORDS[keyword] then
errorf(2, "Bad preprocessor keyword '%s'.", keyword)
else
return { type = "pp_keyword", representation = "@" .. keyword, value = keyword }
end
elseif tokType == "pp_symbol" then
local ident = ...
assertarg(2, ident, "string")
if ident == "" then
error("Identifier length is 0.", 2)
elseif not ident:find("^[%a_][%w_]*$") then
errorf(2, "Bad identifier format: '%s'", ident)
elseif KEYWORDS[ident] then
errorf(2, "Identifier must not be a keyword: '%s'", ident)
else
return { type = "pp_symbol", representation = "$" .. ident, value = ident }
end
else
errorf(2, "Invalid token type '%s'.", tostring(tokType))
end
end
-- concatTokens()
-- luaString = concatTokens( tokens )
-- Concatenate tokens by their representations.
function metaFuncs.concatTokens(tokens)
return (_concatTokens(tokens, nil, false, nil, nil))
end
local recycledArrays = {}
-- startInterceptingOutput()
-- startInterceptingOutput( )
-- Start intercepting output until stopInterceptingOutput() is called.
-- The function can be called multiple times to intercept interceptions.
function metaFuncs.startInterceptingOutput()
errorIfNotRunningMeta(2)
current_meta_output = tableRemove(recycledArrays) or {}
for i = 1, #current_meta_output do
current_meta_output[i] = nil
end
tableInsert(current_meta_outputStack, current_meta_output)
end
local function _stopInterceptingOutput(errLevel)
errorIfNotRunningMeta(1 + errLevel)
local interceptedLua = tableRemove(current_meta_outputStack)
current_meta_output = current_meta_outputStack[#current_meta_outputStack]
or error("Called stopInterceptingOutput() before calling startInterceptingOutput().", 1 + errLevel)
tableInsert(recycledArrays, interceptedLua)
return table.concat(interceptedLua)
end
-- stopInterceptingOutput()
-- luaString = stopInterceptingOutput( )
-- Stop intercepting output and retrieve collected code.
function metaFuncs.stopInterceptingOutput()
return (_stopInterceptingOutput(2))
end
-- loadResource()
-- luaString = loadResource( name )
-- Load a Lua file/resource (using the same mechanism as @insert"name").
-- Note that resources are cached after loading once.
function metaFuncs.loadResource(resourceName)
errorIfNotRunningMeta(2)
return (_loadResource(resourceName, false, 2))
end
local function isCallable(v)
return type(v) == "function"
-- We use debug.getmetatable instead of _G.getmetatable because we don't want to
-- potentially invoke user code - we just want to know if the value is callable.
or (type(v) == "table" and debug.getmetatable(v) ~= nil and type(debug.getmetatable(v).__call) == "function")
end
-- callMacro()
-- luaString = callMacro( function|macroName, argument1, ... )
-- Call a macro function (which must be a global in metaEnvironment if macroName is given).
-- The arguments should be Lua code strings.
function metaFuncs.callMacro(nameOrFunc, ...)
errorIfNotRunningMeta(2)
assertarg(1, nameOrFunc, "string", "function")
local f
if type(nameOrFunc) == "string" then
local nameResult = current_parsingAndMeta_macroPrefix .. nameOrFunc .. current_parsingAndMeta_macroSuffix
f = metaEnv[nameResult]
if not isCallable(f) then
if nameOrFunc == nameResult then
errorf(2, "'%s' is not a macro/global function. (Got %s)", nameOrFunc, type(f))
else
errorf(
2,
"'%s' (resolving to '%s') is not a macro/global function. (Got %s)",
nameOrFunc,
nameResult,
type(f)
)
end
end
else
f = nameOrFunc
end
return (metaEnv.__M()(f(...)))
end
-- isProcessing()
-- bool = isProcessing( )
-- Returns true if a file or string is currently being processed.
function metaFuncs.isProcessing()
return current_parsingAndMeta_isProcessing
end
-- :PredefinedMacros
-- ASSERT()
-- @@ASSERT( condition [, message=auto ] )
-- Macro. Does nothing if params.release is set, otherwise calls error() if the
-- condition fails. The message argument is only evaluated if the condition fails.
function metaFuncs.ASSERT(conditionCode, messageCode)
errorIfNotRunningMeta(2)
if not conditionCode then
error("missing argument #1 to 'ASSERT'", 2)
end
-- if not isLuaStringValidExpression(conditionCode) then
-- errorf(2, "Invalid condition expression: %s", formatCodeForShortMessage(conditionCode))
-- end
if current_meta_releaseMode then
return
end
tableInsert(current_meta_output, "if not (")
tableInsert(current_meta_output, conditionCode)
tableInsert(current_meta_output, ") then error(")
if messageCode then
tableInsert(current_meta_output, "(")
tableInsert(current_meta_output, messageCode)
tableInsert(current_meta_output, ")")
else
tableInsert(current_meta_output, F("%q", "Assertion failed: " .. conditionCode))
end
tableInsert(current_meta_output, ") end")
end
-- LOG()
-- @@LOG( logLevel, value ) -- [1]
-- @@LOG( logLevel, format, value1, ... ) -- [2]
--
-- Macro. Does nothing if logLevel is lower than params.logLevel,
-- otherwise prints a value[1] or a formatted message[2].
--
-- logLevel can be "error", "warning", "info", "debug" or "trace"
-- (from highest to lowest priority).
--
function metaFuncs.LOG(logLevelCode, valueOrFormatCode, ...)
errorIfNotRunningMeta(2)
if not logLevelCode then
error("missing argument #1 to 'LOG'", 2)
end
if not valueOrFormatCode then
error("missing argument #2 to 'LOG'", 2)
end
local chunk = loadLuaString("return(" .. logLevelCode .. "\n)", "@", dummyEnv)
if not chunk then
errorf(2, "Invalid logLevel expression: %s", formatCodeForShortMessage(logLevelCode))
end
local ok, logLevel = pcall(chunk)
if not ok then
errorf(2, "logLevel must be a constant expression. Got: %s", formatCodeForShortMessage(logLevelCode))
end
if not LOG_LEVELS[logLevel] then
errorf(2, "Invalid logLevel '%s'.", tostring(logLevel))
end
if logLevel == "off" then
errorf(2, "Invalid logLevel '%s'.", tostring(logLevel))
end
if LOG_LEVELS[logLevel] > LOG_LEVELS[current_meta_maxLogLevel] then
return
end
tableInsert(current_meta_output, "print(")
if ... then
tableInsert(current_meta_output, "string.format(")
tableInsert(current_meta_output, valueOrFormatCode)
for i = 1, select("#", ...) do
tableInsert(current_meta_output, ", ")
tableInsert(current_meta_output, (select(i, ...)))
end
tableInsert(current_meta_output, ")")
else
tableInsert(current_meta_output, valueOrFormatCode)
end
tableInsert(current_meta_output, ")")
end
-- Extra stuff used by the command line program:
metaFuncs.tryToFormatError = tryToFormatError
----------------------------------------------------------------
for k, v in pairs(metaFuncs) do
metaEnv[k] = v
end
metaEnv.__LUA = metaEnv.outputLua
metaEnv.__VAL = metaEnv.outputValue
function metaEnv.__TOLUA(v)
return (assert(toLua(v)))
end
function metaEnv.__ISLUA(lua)
if type(lua) ~= "string" then
error("Value is not Lua code.", 2)
end
return lua
end
local function finalizeMacro(lua)
if lua == nil then
return (_stopInterceptingOutput(2))
elseif type(lua) ~= "string" then
errorf(2, "[Macro] Value is not Lua code. (Got %s)", type(lua))
elseif current_meta_output[1] then
error("[Macro] Got Lua code from both value expression and outputLua(). Only one method may be used.", 2) -- It's also possible interception calls are unbalanced.
else
_stopInterceptingOutput(2) -- Returns "" because nothing was outputted.
return lua
end
end
function metaEnv.__M()
metaFuncs.startInterceptingOutput()
return finalizeMacro
end
-- luaString = __ARG( locationTokenNumber, luaString|callback )
-- callback = function( )
function metaEnv.__ARG(locTokNum, v)
local lua
if type(v) == "string" then
lua = v
else
metaFuncs.startInterceptingOutput()
v()
lua = _stopInterceptingOutput(2)
end
if current_parsingAndMeta_strictMacroArguments and not isLuaStringValidExpression(lua) then
runtimeErrorAtToken(
2,
current_meta_locationTokens[locTokNum],
nil,
"MacroArgument",
"Argument result is not a valid Lua expression: %s",
formatCodeForShortMessage(lua)
)
end
return lua
end
function metaEnv.__EVAL(v) -- For symbols.
if isCallable(v) then
v = v()
end
return v
end
local function getLineCountWithCode(tokens)
local lineCount = 0
local lastLine = 0
for _, tok in ipairs(tokens) do
if not USELESS_TOKENS[tok.type] and tok.lineEnd > lastLine then
lineCount = lineCount + (tok.lineEnd - tok.line + 1)
lastLine = tok.lineEnd
end
end
return lineCount
end
--
-- Preprocessor expansions (symbols etc., not macros).
--
local function newTokenAt(tok, locTok)
tok.line = tok.line or locTok and locTok.line
tok.lineEnd = tok.lineEnd or locTok and locTok.lineEnd
tok.position = tok.position or locTok and locTok.position
tok.file = tok.file or locTok and locTok.file
return tok
end
local function popTokens(tokenStack, lastIndexToPop)
for i = #tokenStack, lastIndexToPop, -1 do
tokenStack[i] = nil
end
end
local function popUseless(tokenStack)
for i = #tokenStack, 1, -1 do
if not USELESS_TOKENS[tokenStack[i].type] then
break
end
tokenStack[i] = nil
end
end
local function advanceToken(tokens)
local tok = tokens[tokens.nextI]
tokens.nextI = tokens.nextI + 1
return tok
end
local function advancePastUseless(tokens)
for i = tokens.nextI, #tokens do
if not USELESS_TOKENS[tokens[i].type] then
break
end
tokens.nextI = i + 1
end
end
-- outTokens = doEarlyExpansions( tokensToExpand, stats )
local function doEarlyExpansions(tokensToExpand, stats)
--
-- Here we expand simple things that makes it easier for
-- doLateExpansions*() to do more elaborate expansions.
--
-- Expand expressions:
-- @file
-- @line
-- ` ... `
-- $symbol
--
local tokenStack = {} -- We process the last token first, and we may push new tokens onto the stack.
local outTokens = {}
for i = #tokensToExpand, 1, -1 do
tableInsert(tokenStack, tokensToExpand[i])
end
while tokenStack[1] do
local tok = tokenStack[#tokenStack]
-- Keyword.
if isToken(tok, "pp_keyword") then
local ppKeywordTok = tok
-- @file
-- @line
if ppKeywordTok.value == "file" then
tableRemove(tokenStack) -- '@file'
tableInsert(
outTokens,
newTokenAt(
{ type = "string", value = ppKeywordTok.file, representation = F("%q", ppKeywordTok.file) },
ppKeywordTok
)
)
elseif ppKeywordTok.value == "line" then
tableRemove(tokenStack) -- '@line'
tableInsert(
outTokens,
newTokenAt(
{ type = "number", value = ppKeywordTok.line, representation = F(" %d ", ppKeywordTok.line) },
ppKeywordTok
)
) -- Is it fine for the representation to have spaces? Probably.
else
-- Expand later.
tableInsert(outTokens, ppKeywordTok)
tableRemove(tokenStack) -- '@...'
end
-- Backtick string.
elseif isToken(tok, "string") and tok.representation:find("^`") then
local stringTok = tok
stringTok.representation = toLua(stringTok.value) --F("%q", stringTok.value)
tableInsert(outTokens, stringTok)
tableRemove(tokenStack) -- the string
-- Symbol. (Should this expand later? Does it matter? Yeah, do this in the AST code instead. @Cleanup)
elseif isToken(tok, "pp_symbol") then
local ppSymbolTok = tok
-- $symbol
tableRemove(tokenStack) -- '$symbol'
tableInsert(
outTokens,
newTokenAt({ type = "pp_entry", value = "!!", representation = "!!", double = true }, ppSymbolTok)
)
tableInsert(
outTokens,
newTokenAt({
type = "punctuation",
value = "(",
representation = "(",
}, ppSymbolTok)
)
tableInsert(
outTokens,
newTokenAt({ type = "identifier", value = "__EVAL", representation = "__EVAL" }, ppSymbolTok)
)
tableInsert(
outTokens,
newTokenAt({
type = "punctuation",
value = "(",
representation = "(",
}, ppSymbolTok)
)
tableInsert(
outTokens,
newTokenAt(
{ type = "identifier", value = ppSymbolTok.value, representation = ppSymbolTok.value },
ppSymbolTok
)
)
tableInsert(
outTokens,
newTokenAt({
type = "punctuation",
value = ")",
representation = ")",
}, ppSymbolTok)
)
tableInsert(
outTokens,
newTokenAt({
type = "punctuation",
value = ")",
representation = ")",
}, ppSymbolTok)
)
-- Anything else.
else
tableInsert(outTokens, tok)
tableRemove(tokenStack) -- anything
end
end --while tokenStack
return outTokens
end
-- outTokens = doLateExpansions( tokensToExpand, stats, allowBacktickStrings, allowJitSyntax )
local function doLateExpansions(tokensToExpand, stats, allowBacktickStrings, allowJitSyntax)
--
-- Expand expressions:
-- @insert "name"
--
local tokenStack = {} -- We process the last token first, and we may push new tokens onto the stack.
local outTokens = {}
for i = #tokensToExpand, 1, -1 do
tableInsert(tokenStack, tokensToExpand[i])
end
while tokenStack[1] do
local tok = tokenStack[#tokenStack]
-- Keyword.
if isToken(tok, "pp_keyword") then
local ppKeywordTok = tok
local tokNext, iNext = getNextUsableToken(tokenStack, #tokenStack - 1, nil, -1)
-- @insert "name"
if
ppKeywordTok.value == "insert"
and isTokenAndNotNil(tokNext, "string")
and tokNext.file == ppKeywordTok.file
then
local nameTok = tokNext
popTokens(tokenStack, iNext) -- the string
local toInsertName = nameTok.value
local toInsertLua = _loadResource(toInsertName, true, nameTok, stats)
local toInsertTokens = _tokenize(toInsertLua, toInsertName, true, allowBacktickStrings, allowJitSyntax)
toInsertTokens = doEarlyExpansions(toInsertTokens, stats)
for i = #toInsertTokens, 1, -1 do
tableInsert(tokenStack, toInsertTokens[i])
end
local lastTok = toInsertTokens[#toInsertTokens]
stats.processedByteCount = stats.processedByteCount + #toInsertLua
stats.lineCount = stats.lineCount
+ (lastTok and lastTok.line + countString(lastTok.representation, "\n", true) or 0)
stats.lineCountCode = stats.lineCountCode + getLineCountWithCode(toInsertTokens)
-- @insert identifier ( argument1, ... )
-- @insert identifier " ... "
-- @insert identifier { ... }
-- @insert identifier !( ... )
-- @insert identifier !!( ... )
elseif
ppKeywordTok.value == "insert"
and isTokenAndNotNil(tokNext, "identifier")
and tokNext.file == ppKeywordTok.file
then
local identTok = tokNext
tokNext, iNext = getNextUsableToken(tokenStack, iNext - 1, nil, -1)
if
not (
tokNext
and (
tokNext.type == "string"
or (tokNext.type == "punctuation" and isAny(tokNext.value, "(", "{", ".", ":", "["))
or tokNext.type == "pp_entry"
)
)
then
errorAtToken(
identTok,
identTok.position + #identTok.representation,
"Parser/Macro",
"Expected '(' after macro name '%s'.",
identTok.value
)
end
-- Expand later.
tableInsert(outTokens, tok)
tableRemove(tokenStack) -- '@insert'
elseif ppKeywordTok.value == "insert" then
errorAtToken(
ppKeywordTok,
(tokNext and tokNext.position or ppKeywordTok.position + #ppKeywordTok.representation),
"Parser",
"Expected a string or identifier after %s.",
ppKeywordTok.representation
)
else
errorAtToken(ppKeywordTok, nil, "Parser", "Internal error. (%s)", ppKeywordTok.value)
end
-- Anything else.
else
tableInsert(outTokens, tok)
tableRemove(tokenStack) -- anything
end
end --while tokenStack
return outTokens
end
-- outTokens = doExpansions( params, tokensToExpand, stats )
local function doExpansions(params, tokens, stats)
tokens = doEarlyExpansions(tokens, stats)
tokens = doLateExpansions(tokens, stats, params.backtickStrings, params.jitSyntax) -- Resources.
return tokens
end
--
-- Metaprogram generation.
--
local function AstSequence(locTok, tokens)
return {
type = "sequence",
locationToken = locTok,
nodes = tokens or {},
}
end
local function AstLua(locTok, tokens)
return { -- plain Lua
type = "lua",
locationToken = locTok,
tokens = tokens or {},
}
end
local function AstMetaprogram(locTok, tokens)
return { -- `!(statements)` or `!statements`
type = "metaprogram",
locationToken = locTok,
originIsLine = false,
tokens = tokens or {},
}
end
local function AstExpressionCode(locTok, tokens)
return { -- `!!(expression)`
type = "expressionCode",
locationToken = locTok,
tokens = tokens or {},
}
end
local function AstExpressionValue(locTok, tokens)
return { -- `!(expression)`
type = "expressionValue",
locationToken = locTok,
tokens = tokens or {},
}
end
local function AstDualCode(locTok, valueTokens)
return { -- `!!declaration` or `!!assignment`
type = "dualCode",
locationToken = locTok,
isDeclaration = false,
names = {},
valueTokens = valueTokens or {},
}
end
-- local function AstSymbol(locTok) return { -- `$name`
-- type = "symbol",
-- locationToken = locTok,
-- name = "",
-- } end
local function AstMacro(locTok, calleeTokens)
return { -- `@@callee(arguments)` or `@@callee{}` or `@@callee""`
type = "macro",
locationToken = locTok,
calleeTokens = calleeTokens or {},
arguments = {}, -- []MacroArgument
}
end
local function MacroArgument(locTok, nodes)
return {
locationToken = locTok,
isComplex = false,
nodes = nodes or {},
}
end
local astParseMetaBlockOrLine
local function astParseMetaBlock(tokens)
local ppEntryTokIndex = tokens.nextI
local ppEntryTok = tokens[ppEntryTokIndex]
tokens.nextI = tokens.nextI + 2 -- '!(' or '!!('
local outTokens = {}
local depthStack = {}
while true do
local tok = tokens[tokens.nextI]
if not tok then
if depthStack[1] then
tok = depthStack[#depthStack].startToken
errorAtToken(
tok,
nil,
"Parser/MetaBlock",
"Could not find matching bracket before EOF. (Preprocessor line starts %s)",
getRelativeLocationText(ppEntryTok, tok)
)
end
break
end
-- End of meta block.
if not depthStack[1] and isToken(tok, "punctuation", ")") then
tokens.nextI = tokens.nextI + 1 -- after ')'
break
-- Nested metaprogram (not supported).
elseif tok.type:find("^pp_") then
errorAtToken(
tok,
nil,
"Parser/MetaBlock",
"Preprocessor token inside metaprogram (starting %s).",
getRelativeLocationText(ppEntryTok, tok)
)
-- Continuation of meta block.
else
if isToken(tok, "punctuation", "(") then
tableInsert(depthStack, {
startToken = tok, --[[1]]
"punctuation", --[[2]]
")",
})
elseif isToken(tok, "punctuation", "[") then
tableInsert(depthStack, {
startToken = tok, --[[1]]
"punctuation", --[[2]]
"]",
})
elseif isToken(tok, "punctuation", "{") then
tableInsert(depthStack, {
startToken = tok, --[[1]]
"punctuation", --[[2]]
"}",
})
elseif
isToken(tok, "punctuation", ")")
or isToken(tok, "punctuation", "]")
or isToken(tok, "punctuation", "}")
then
if not depthStack[1] then
errorAtToken(
tok,
nil,
"Parser/MetaBlock",
"Unexpected '%s'. (Preprocessor line starts %s)",
tok.value,
getRelativeLocationText(ppEntryTok, tok)
)
elseif not isToken(tok, unpack(depthStack[#depthStack])) then
local startTok = depthStack[#depthStack].startToken
errorAtToken(
tok,
nil,
"Parser/MetaBlock",
"Expected '%s' (to close '%s' %s) but got '%s'. (Preprocessor line starts %s)",
depthStack[#depthStack][2],
startTok.value,
getRelativeLocationText(startTok, tok),
tok.value,
getRelativeLocationText(ppEntryTok, tok)
)
end
tableRemove(depthStack)
end
tableInsert(outTokens, tok)
tokens.nextI = tokens.nextI + 1 -- after anything
end
end
local lua = _concatTokens(outTokens, nil, false, nil, nil)
local chunk, err = loadLuaString("return 0," .. lua .. "\n,0", "@", nil)
local isExpression = (chunk ~= nil)
if not isExpression and ppEntryTok.double then
errorAtToken(tokens[ppEntryTokIndex + 1], nil, "Parser/MetaBlock", "Invalid expression in preprocessor block.")
-- err = err:gsub("^:%d+: ", "")
-- errorAtToken(tokens[ppEntryTokIndex+1], nil, "Parser/MetaBlock", "Invalid expression in preprocessor block. (%s)", err)
elseif isExpression and not isLuaStringValidExpression(lua) then
if #lua > 100 then
lua = lua:sub(1, 50) .. "..." .. lua:sub(-50)
end
errorAtToken(
tokens[ppEntryTokIndex + 1],
nil,
"Parser/MetaBlock",
"Ambiguous expression '%s'. (Comma-separated list?)",
formatCodeForShortMessage(lua)
)
end
local astOutNode = (
(ppEntryTok.double and AstExpressionCode) or (isExpression and AstExpressionValue or AstMetaprogram)
)(ppEntryTok, outTokens)
return astOutNode
end
local function astParseMetaLine(tokens)
local ppEntryTok = tokens[tokens.nextI]
tokens.nextI = tokens.nextI + 1 -- '!' or '!!'
local isDual = ppEntryTok.double
local astOutNode = (isDual and AstDualCode or AstMetaprogram)(ppEntryTok)
if astOutNode.type == "metaprogram" then
astOutNode.originIsLine = true
end
if isDual then
-- We expect the statement to look like any of these:
-- !!local x, y = ...
-- !!x, y = ...
local tokNext, iNext = getNextUsableToken(tokens, tokens.nextI, nil, 1)
if isTokenAndNotNil(tokNext, "keyword", "local") then
astOutNode.isDeclaration = true
tokens.nextI = iNext + 1 -- after 'local'
tokNext, iNext = getNextUsableToken(tokens, tokens.nextI, nil, 1)
end
local usedNames = {}
while true do
if not isTokenAndNotNil(tokNext, "identifier") then
local tok = tokNext or tokens[#tokens]
errorAtToken(
tok,
nil,
"Parser/DualCodeLine",
"Expected %sidentifier. (Preprocessor line starts %s)",
(astOutNode.names[1] and "" or "'local' or "),
getRelativeLocationText(ppEntryTok, tok)
)
elseif usedNames[tokNext.value] then
errorAtToken(
tokNext,
nil,
"Parser/DualCodeLine",
"Duplicate name '%s' in %s. (Preprocessor line starts %s)",
tokNext.value,
(astOutNode.isDeclaration and "declaration" or "assignment"),
getRelativeLocationText(ppEntryTok, tokNext)
)
end
tableInsert(astOutNode.names, tokNext.value)
usedNames[tokNext.value] = tokNext
tokens.nextI = iNext + 1 -- after the identifier
tokNext, iNext = getNextUsableToken(tokens, tokens.nextI, nil, 1)
if not isTokenAndNotNil(tokNext, "punctuation", ",") then
break
end
tokens.nextI = iNext + 1 -- after ','
tokNext, iNext = getNextUsableToken(tokens, tokens.nextI, nil, 1)
end
if not isTokenAndNotNil(tokNext, "punctuation", "=") then
local tok = tokNext or tokens[#tokens]
errorAtToken(
tok,
nil,
"Parser/DualCodeLine",
"Expected '=' in %s. (Preprocessor line starts %s)",
(astOutNode.isDeclaration and "declaration" or "assignment"),
getRelativeLocationText(ppEntryTok, tok)
)
end
tokens.nextI = iNext + 1 -- after '='
end
-- Find end of metaprogram line.
local outTokens = isDual and astOutNode.valueTokens or astOutNode.tokens
local depthStack = {}
while true do
local tok = tokens[tokens.nextI]
if not tok then
if depthStack[1] then
tok = depthStack[#depthStack].startToken
errorAtToken(
tok,
nil,
"Parser/MetaLine",
"Could not find matching bracket before EOF. (Preprocessor line starts %s)",
getRelativeLocationText(ppEntryTok, tok)
)
end
break
end
-- End of meta line.
if
not depthStack[1]
and (
(tok.type == "whitespace" and tok.value:find("\n", 1, true))
or (tok.type == "comment" and not tok.long)
)
then
tableInsert(outTokens, tok)
tokens.nextI = tokens.nextI + 1 -- after the whitespace or comment
break
-- Nested metaprogram (not supported).
elseif tok.type:find("^pp_") then
errorAtToken(
tok,
nil,
"Parser/MetaLine",
"Preprocessor token inside metaprogram (starting %s).",
getRelativeLocationText(ppEntryTok, tok)
)
-- Continuation of meta line.
else
if isToken(tok, "punctuation", "(") then
tableInsert(depthStack, {
startToken = tok, --[[1]]
"punctuation", --[[2]]
")",
})
elseif isToken(tok, "punctuation", "[") then
tableInsert(depthStack, {
startToken = tok, --[[1]]
"punctuation", --[[2]]
"]",
})
elseif isToken(tok, "punctuation", "{") then
tableInsert(depthStack, {
startToken = tok, --[[1]]
"punctuation", --[[2]]
"}",
})
elseif
isToken(tok, "punctuation", ")")
or isToken(tok, "punctuation", "]")
or isToken(tok, "punctuation", "}")
then
if not depthStack[1] then
errorAtToken(
tok,
nil,
"Parser/MetaLine",
"Unexpected '%s'. (Preprocessor line starts %s)",
tok.value,
getRelativeLocationText(ppEntryTok, tok)
)
elseif not isToken(tok, unpack(depthStack[#depthStack])) then
local startTok = depthStack[#depthStack].startToken
errorAtToken(
tok,
nil,
"Parser/MetaLine",
"Expected '%s' (to close '%s' %s) but got '%s'. (Preprocessor line starts %s)",
depthStack[#depthStack][2],
startTok.value,
getRelativeLocationText(startTok, tok),
tok.value,
getRelativeLocationText(ppEntryTok, tok)
)
end
tableRemove(depthStack)
end
tableInsert(outTokens, tok)
tokens.nextI = tokens.nextI + 1 -- after anything
end
end
return astOutNode
end
--[[local]]
function astParseMetaBlockOrLine(tokens)
return isTokenAndNotNil(tokens[tokens.nextI + 1], "punctuation", "(") and astParseMetaBlock(tokens)
or astParseMetaLine(tokens)
end
local function astParseMacro(params, tokens)
local macroStartTok = tokens[tokens.nextI]
tokens.nextI = tokens.nextI + 1 -- after '@insert'
local astMacro = AstMacro(macroStartTok)
--
-- Callee.
--
-- Add 'ident' for start of (or whole) callee.
local tokNext, iNext = getNextUsableToken(tokens, tokens.nextI, nil, 1)
if not isTokenAndNotNil(tokNext, "identifier") then
printErrorTraceback("Internal error.")
errorAtToken(tokNext, nil, "Parser/Macro", "Internal error. (%s)", (tokNext and tokNext.type or "?"))
end
tokens.nextI = iNext + 1 -- after the identifier
tableInsert(astMacro.calleeTokens, tokNext)
local initialCalleeIdentTok = tokNext
-- Add macro prefix and suffix. (Note: We only edit the initial identifier in the callee if there are more.)
initialCalleeIdentTok.value = current_parsingAndMeta_macroPrefix
.. initialCalleeIdentTok.value
.. current_parsingAndMeta_macroSuffix
initialCalleeIdentTok.representation = initialCalleeIdentTok.value
-- Maybe add '.field[expr]:method' for rest of callee.
tokNext, iNext = getNextUsableToken(tokens, tokens.nextI, nil, 1)
while tokNext do
if isToken(tokNext, "punctuation", ".") or isToken(tokNext, "punctuation", ":") then
local punctTok = tokNext
tokens.nextI = iNext + 1 -- after '.' or ':'
tableInsert(astMacro.calleeTokens, tokNext)
tokNext, iNext = getNextUsableToken(tokens, tokens.nextI, nil, 1)
if not tokNext then
errorAfterToken(punctTok, "Parser/Macro", "Expected an identifier after '%s'.", punctTok.value)
end
tokens.nextI = iNext + 1 -- after the identifier
tableInsert(astMacro.calleeTokens, tokNext)
tokNext, iNext = getNextUsableToken(tokens, tokens.nextI, nil, 1)
if punctTok.value == ":" then
break
end
elseif isToken(tokNext, "punctuation", "[") then
local punctTok = tokNext
tokens.nextI = iNext + 1 -- after '['
tableInsert(astMacro.calleeTokens, tokNext)
local bracketBalance = 1
while true do
tokNext = advanceToken(tokens) -- anything
if not tokNext then
errorAtToken(
punctTok,
nil,
"Parser/Macro",
"Could not find matching bracket before EOF. (Macro starts %s)",
getRelativeLocationText(macroStartTok, punctTok)
)
end
tableInsert(astMacro.calleeTokens, tokNext)
if isToken(tokNext, "punctuation", "[") then
bracketBalance = bracketBalance + 1
elseif isToken(tokNext, "punctuation", "]") then
bracketBalance = bracketBalance - 1
if bracketBalance == 0 then
break
end
elseif tokNext.type:find("^pp_") then
errorAtToken(
tokNext,
nil,
"Parser/Macro",
"Preprocessor token inside metaprogram/macro name expression (starting %s).",
getRelativeLocationText(macroStartTok, tokNext)
)
end
end
tokNext, iNext = getNextUsableToken(tokens, tokens.nextI, nil, 1)
-- @UX: Validate that the contents form an expression.
else
break
end
end
--
-- Arguments.
--
-- @insert identifier " ... "
if isTokenAndNotNil(tokNext, "string") then
tableInsert(astMacro.arguments, MacroArgument(tokNext, { AstLua(tokNext, { tokNext }) })) -- The one and only argument for this macro variant.
tokens.nextI = iNext + 1 -- after the string
-- @insert identifier { ... } -- Same as: @insert identifier ( { ... } )
elseif isTokenAndNotNil(tokNext, "punctuation", "{") then
local macroArg = MacroArgument(tokNext) -- The one and only argument for this macro variant.
astMacro.arguments[1] = macroArg
local astLuaInCurrentArg = AstLua(tokNext, { tokNext })
tableInsert(macroArg.nodes, astLuaInCurrentArg)
tokens.nextI = iNext + 1 -- after '{'
--
-- (Similar code as `@insert identifier()` below.)
--
-- Collect tokens for the table arg.
-- We're looking for the closing '}'.
local bracketDepth = 1 -- @Incomplete: Track all brackets!
while true do
local tok = tokens[tokens.nextI]
if not tok then
errorAtToken(
macroArg.locationToken,
nil,
"Parser/MacroArgument",
"Could not find end of table constructor before EOF."
)
-- Preprocessor block in macro.
elseif tok.type == "pp_entry" then
tableInsert(macroArg.nodes, astParseMetaBlockOrLine(tokens))
astLuaInCurrentArg = nil
-- Nested macro.
elseif isToken(tok, "pp_keyword", "insert") then
tableInsert(macroArg.nodes, astParseMacro(params, tokens))
astLuaInCurrentArg = nil
-- Other preprocessor code in macro. (Not sure we ever get here.)
elseif tok.type:find("^pp_") then
errorAtToken(
tok,
nil,
"Parser/MacroArgument",
"Unsupported preprocessor code. (Macro starts %s)",
getRelativeLocationText(macroStartTok, tok)
)
-- End of table and argument.
elseif bracketDepth == 1 and isToken(tok, "punctuation", "}") then
if not astLuaInCurrentArg then
astLuaInCurrentArg = AstLua(tok)
tableInsert(macroArg.nodes, astLuaInCurrentArg)
end
tableInsert(astLuaInCurrentArg.tokens, tok)
advanceToken(tokens) -- '}'
break
-- Normal token.
else
if isToken(tok, "punctuation", "{") then
bracketDepth = bracketDepth + 1
elseif isToken(tok, "punctuation", "}") then
bracketDepth = bracketDepth - 1
end
if not astLuaInCurrentArg then
astLuaInCurrentArg = AstLua(tok)
tableInsert(macroArg.nodes, astLuaInCurrentArg)
end
tableInsert(astLuaInCurrentArg.tokens, tok)
advanceToken(tokens) -- anything
end
end
-- @insert identifier ( argument1, ... )
elseif isTokenAndNotNil(tokNext, "punctuation", "(") then
-- Apply the same 'ambiguous syntax' rule as Lua. (Will comments mess this check up? @Check)
if isTokenAndNotNil(tokens[iNext - 1], "whitespace") and tokens[iNext - 1].value:find("\n", 1, true) then
errorAtToken(tokNext, nil, "Parser/Macro", "Ambiguous syntax near '(' - part of macro, or new statement?")
end
local parensStartTok = tokNext
tokens.nextI = iNext + 1 -- after '('
tokNext, iNext = getNextUsableToken(tokens, tokens.nextI, nil, 1)
if isTokenAndNotNil(tokNext, "punctuation", ")") then
tokens.nextI = iNext + 1 -- after ')'
else
for argNum = 1, 1 / 0 do
-- Collect tokens for this arg.
-- We're looking for the next comma at depth 0 or closing ')'.
local macroArg = MacroArgument(tokens[tokens.nextI])
astMacro.arguments[argNum] = macroArg
advancePastUseless(tokens) -- Trim leading useless tokens.
local astLuaInCurrentArg = nil
local depthStack = {}
while true do
local tok = tokens[tokens.nextI]
if not tok then
errorAtToken(
parensStartTok,
nil,
"Parser/Macro",
"Could not find end of argument list before EOF."
)
-- Preprocessor block in macro.
elseif tok.type == "pp_entry" then
tableInsert(macroArg.nodes, astParseMetaBlockOrLine(tokens))
astLuaInCurrentArg = nil
-- Nested macro.
elseif isToken(tok, "pp_keyword", "insert") then
tableInsert(macroArg.nodes, astParseMacro(params, tokens))
astLuaInCurrentArg = nil
-- Other preprocessor code in macro. (Not sure we ever get here.)
elseif tok.type:find("^pp_") then
errorAtToken(
tok,
nil,
"Parser/MacroArgument",
"Unsupported preprocessor code. (Macro starts %s)",
getRelativeLocationText(macroStartTok, tok)
)
-- End of argument.
elseif
not depthStack[1] and (isToken(tok, "punctuation", ",") or isToken(tok, "punctuation", ")"))
then
break
-- Normal token.
else
if isToken(tok, "punctuation", "(") then
tableInsert(depthStack, {
startToken = tok, --[[1]]
"punctuation", --[[2]]
")",
})
elseif isToken(tok, "punctuation", "[") then
tableInsert(depthStack, {
startToken = tok, --[[1]]
"punctuation", --[[2]]
"]",
})
elseif isToken(tok, "punctuation", "{") then
tableInsert(depthStack, {
startToken = tok, --[[1]]
"punctuation", --[[2]]
"}",
})
elseif
isToken(tok, "keyword", "function")
or isToken(tok, "keyword", "if")
or isToken(tok, "keyword", "do")
then
tableInsert(depthStack, {
startToken = tok, --[[1]]
"keyword", --[[2]]
"end",
})
elseif isToken(tok, "keyword", "repeat") then
tableInsert(depthStack, {
startToken = tok, --[[1]]
"keyword", --[[2]]
"until",
})
elseif
isToken(tok, "punctuation", ")")
or isToken(tok, "punctuation", "]")
or isToken(tok, "punctuation", "}")
or isToken(tok, "keyword", "end")
or isToken(tok, "keyword", "until")
then
if not depthStack[1] then
errorAtToken(tok, nil, "Parser/MacroArgument", "Unexpected '%s'.", tok.value)
elseif not isToken(tok, unpack(depthStack[#depthStack])) then
local startTok = depthStack[#depthStack].startToken
errorAtToken(
tok,
nil,
"Parser/MacroArgument",
"Expected '%s' (to close '%s' %s) but got '%s'.",
depthStack[#depthStack][2],
startTok.value,
getRelativeLocationText(startTok, tok),
tok.value
)
end
tableRemove(depthStack)
end
if not astLuaInCurrentArg then
astLuaInCurrentArg = AstLua(tok)
tableInsert(macroArg.nodes, astLuaInCurrentArg)
end
tableInsert(astLuaInCurrentArg.tokens, tok)
advanceToken(tokens) -- anything
end
end
if astLuaInCurrentArg then
-- Trim trailing useless tokens.
popUseless(astLuaInCurrentArg.tokens)
if not astLuaInCurrentArg.tokens[1] then
assert(tableRemove(macroArg.nodes) == astLuaInCurrentArg)
end
end
if not macroArg.nodes[1] and current_parsingAndMeta_strictMacroArguments then
-- There were no useful tokens for the argument!
errorAtToken(macroArg.locationToken, nil, "Parser/MacroArgument", "Expected argument #%d.", argNum)
end
-- Do next argument or finish arguments.
if isTokenAndNotNil(tokens[tokens.nextI], "punctuation", ")") then
tokens.nextI = tokens.nextI + 1 -- after ')'
break
end
assert(isToken(advanceToken(tokens), "punctuation", ",")) -- The loop above should have continued otherwise!
end --for argNum
end
-- @insert identifier !( ... ) -- Same as: @insert identifier ( !( ... ) )
-- @insert identifier !!( ... ) -- Same as: @insert identifier ( !!( ... ) )
elseif isTokenAndNotNil(tokNext, "pp_entry") then
tokens.nextI = iNext -- until '!' or '!!'
if not isTokenAndNotNil(tokens[tokens.nextI + 1], "punctuation", "(") then
errorAfterToken(tokNext, "Parser/Macro", "Expected '(' after '%s'.", tokNext.value)
end
astMacro.arguments[1] = MacroArgument(tokNext, { astParseMetaBlock(tokens) }) -- The one and only argument for this macro variant.
else
errorAfterToken(astMacro.calleeTokens[#astMacro.calleeTokens], "Parser/Macro", "Expected '(' after macro name.")
end
return astMacro
end
local function astParse(params, tokens)
-- @Robustness: Make sure everywhere that key tokens came from the same source file.
local astSequence = AstSequence(tokens[1])
tokens.nextI = 1
while true do
local tok = tokens[tokens.nextI]
if not tok then
break
end
if isToken(tok, "pp_entry") then
tableInsert(astSequence.nodes, astParseMetaBlockOrLine(tokens))
elseif isToken(tok, "pp_keyword", "insert") then
local astMacro = astParseMacro(params, tokens)
tableInsert(astSequence.nodes, astMacro)
-- elseif isToken(tok, "pp_symbol") then -- We currently expand these in doEarlyExpansions().
-- errorAtToken(tok, nil, "Parser", "Internal error: @Incomplete: Handle symbols.")
else
local astLua = AstLua(tok)
tableInsert(astSequence.nodes, astLua)
while true do
tableInsert(astLua.tokens, tok)
advanceToken(tokens)
tok = tokens[tokens.nextI]
if not tok then
break
end
if tok.type:find("^pp_") then
break
end
end
end
end
return astSequence
end
-- lineNumber, lineNumberMeta = astNodeToMetaprogram( buffer, ast, lineNumber, lineNumberMeta, asMacroArgumentExpression )
local function astNodeToMetaprogram(buffer, ast, ln, lnMeta, asMacroArgExpr)
if current_parsingAndMeta_addLineNumbers and not asMacroArgExpr then
lnMeta = maybeOutputLineNumber(buffer, ast.locationToken, lnMeta)
end
--
-- lua -> __LUA"lua"
--
if ast.type == "lua" then
local lua = _concatTokens(ast.tokens, ln, current_parsingAndMeta_addLineNumbers, nil, nil)
ln = ast.tokens[#ast.tokens].line
if not asMacroArgExpr then
tableInsert(buffer, "__LUA")
end
if current_parsingAndMeta_isDebug then
if not asMacroArgExpr then
tableInsert(buffer, "(")
end
tableInsert(buffer, (F("%q", lua):gsub("\n", "n")))
if not asMacroArgExpr then
tableInsert(buffer, ")\n")
end
else
tableInsert(buffer, F("%q", lua))
if not asMacroArgExpr then
tableInsert(buffer, "\n")
end
end
--
-- !(expression) -> __VAL(expression)
--
elseif ast.type == "expressionValue" then
if asMacroArgExpr then
tableInsert(buffer, "__TOLUA(")
else
tableInsert(buffer, "__VAL((")
end
for _, tok in ipairs(ast.tokens) do
tableInsert(buffer, tok.representation)
end
if asMacroArgExpr then
tableInsert(buffer, ")")
else
tableInsert(buffer, "))\n")
end
--
-- !!(expression) -> __LUA(expression)
--
elseif ast.type == "expressionCode" then
if asMacroArgExpr then
tableInsert(buffer, "__ISLUA(")
else
tableInsert(buffer, "__LUA((")
end
for _, tok in ipairs(ast.tokens) do
tableInsert(buffer, tok.representation)
end
if asMacroArgExpr then
tableInsert(buffer, ")")
else
tableInsert(buffer, "))\n")
end
--
-- !(statements) -> statements
-- !statements -> statements
--
elseif ast.type == "metaprogram" then
if asMacroArgExpr then
internalError(ast.type)
end
if ast.originIsLine then
for i = 1, #ast.tokens - 1 do
tableInsert(buffer, ast.tokens[i].representation)
end
local lastTok = ast.tokens[#ast.tokens]
if lastTok.type == "whitespace" then
if current_parsingAndMeta_isDebug then
tableInsert(buffer, (F("\n__LUA(%q)\n", lastTok.value):gsub("\\\n", "\\n"))) -- Note: "\\\n" does not match "\n".
else
tableInsert(buffer, (F("\n__LUA%q\n", lastTok.value):gsub("\\\n", "\\n")))
end
else --if type == comment
tableInsert(buffer, lastTok.representation)
if current_parsingAndMeta_isDebug then
tableInsert(buffer, F('__LUA("\\n")\n'))
else
tableInsert(buffer, F("__LUA'\\n'\n"))
end
end
else
for _, tok in ipairs(ast.tokens) do
tableInsert(buffer, tok.representation)
end
tableInsert(buffer, "\n")
end
--
-- @@callee(argument1, ...) -> __LUA(__M(callee(__ARG(1,<argument1>), ...)))
-- OR -> __LUA(__M(callee(__ARG(1,function()<argument1>end), ...)))
--
-- The code handling each argument will be different depending on the complexity of the argument.
--
elseif ast.type == "macro" then
if not asMacroArgExpr then
tableInsert(buffer, "__LUA(")
end
tableInsert(buffer, "__M()(")
for _, tok in ipairs(ast.calleeTokens) do
tableInsert(buffer, tok.representation)
end
tableInsert(buffer, "(")
for argNum, macroArg in ipairs(ast.arguments) do
local argIsComplex = false -- If any part of the argument cannot be an expression then it's complex.
for _, astInArg in ipairs(macroArg.nodes) do
if astInArg.type == "metaprogram" or astInArg.type == "dualCode" then
argIsComplex = true
break
end
end
if argNum > 1 then
tableInsert(buffer, ",")
if current_parsingAndMeta_isDebug then
tableInsert(buffer, " ")
end
end
local locTokNum = #current_meta_locationTokens + 1
current_meta_locationTokens[locTokNum] = macroArg.nodes[1] and macroArg.nodes[1].locationToken
or macroArg.locationToken
or internalError()
tableInsert(buffer, "__ARG(")
tableInsert(buffer, tostring(locTokNum))
tableInsert(buffer, ",")
if argIsComplex then
tableInsert(buffer, "function()\n")
for nodeNumInArg, astInArg in ipairs(macroArg.nodes) do
ln, lnMeta = astNodeToMetaprogram(buffer, astInArg, ln, lnMeta, false)
end
tableInsert(buffer, "end")
elseif macroArg.nodes[1] then
for nodeNumInArg, astInArg in ipairs(macroArg.nodes) do
if nodeNumInArg > 1 then
tableInsert(buffer, "..")
end
ln, lnMeta = astNodeToMetaprogram(buffer, astInArg, ln, lnMeta, true)
end
else
tableInsert(buffer, '""')
end
tableInsert(buffer, ")")
end
tableInsert(buffer, "))")
if not asMacroArgExpr then
tableInsert(buffer, ")\n")
end
--
-- !!local names = values -> local names = values ; __LUA"local names = "__VAL(name1)__LUA", "__VAL(name2)...
-- !! names = values -> names = values ; __LUA"names = "__VAL(name1)__LUA", "__VAL(name2)...
--
elseif ast.type == "dualCode" then
if asMacroArgExpr then
internalError(ast.type)
end
-- Metaprogram.
if ast.isDeclaration then
tableInsert(buffer, "local ")
end
tableInsert(buffer, table.concat(ast.names, ", "))
tableInsert(buffer, " = ")
for _, tok in ipairs(ast.valueTokens) do
tableInsert(buffer, tok.representation)
end
-- Final program.
tableInsert(buffer, "__LUA")
if current_parsingAndMeta_isDebug then
tableInsert(buffer, "(")
end
tableInsert(buffer, '"') -- string start
if current_parsingAndMeta_addLineNumbers then
ln = maybeOutputLineNumber(buffer, ast.locationToken, ln)
end
if ast.isDeclaration then
tableInsert(buffer, "local ")
end
tableInsert(buffer, table.concat(ast.names, ", "))
tableInsert(buffer, ' = "') -- string end
if current_parsingAndMeta_isDebug then
tableInsert(buffer, "); ")
end
for i, name in ipairs(ast.names) do
if i == 1 then -- void
elseif current_parsingAndMeta_isDebug then
tableInsert(buffer, '; __LUA(", "); ')
else
tableInsert(buffer, '__LUA", "')
end
tableInsert(buffer, "__VAL(")
tableInsert(buffer, name)
tableInsert(buffer, ")")
end
-- Use trailing semicolon if the user does.
for i = #ast.valueTokens, 1, -1 do
if isToken(ast.valueTokens[i], "punctuation", ";") then
if current_parsingAndMeta_isDebug then
tableInsert(buffer, '; __LUA(";")')
else
tableInsert(buffer, '__LUA";"')
end
break
elseif not isToken(ast.valueTokens[i], "whitespace") then
break
end
end
if current_parsingAndMeta_isDebug then
tableInsert(buffer, '; __LUA("\\n")\n')
else
tableInsert(buffer, '__LUA"\\n"\n')
end
--
-- ...
--
elseif ast.type == "sequence" then
for _, astChild in ipairs(ast.nodes) do
ln, lnMeta = astNodeToMetaprogram(buffer, astChild, ln, lnMeta, false)
end
-- elseif ast.type == "symbol" then
-- errorAtToken(ast.locationToken, nil, nil, "AstSymbol")
else
printErrorTraceback("Internal error.")
errorAtToken(ast.locationToken, nil, "Parsing", "Internal error. (%s, %s)", ast.type, tostring(asMacroArgExpr))
end
return ln, lnMeta
end
local function astToLua(ast)
local buffer = {}
astNodeToMetaprogram(buffer, ast, 0, 0, false)
return table.concat(buffer)
end
local function _processFileOrString(params, isFile)
if isFile then
if not params.pathIn then
error("Missing 'pathIn' in params.", 2)
end
if not params.pathOut then
error("Missing 'pathOut' in params.", 2)
end
if params.pathOut == params.pathIn and params.pathOut ~= "-" then
error("'pathIn' and 'pathOut' are the same in params.", 2)
end
if (params.pathMeta or "-") == "-" then -- Should it be possible to output the metaprogram to stdout?
-- void
elseif params.pathMeta == params.pathIn then
error("'pathIn' and 'pathMeta' are the same in params.", 2)
elseif params.pathMeta == params.pathOut then
error("'pathOut' and 'pathMeta' are the same in params.", 2)
end
else
if not params.code then
error("Missing 'code' in params.", 2)
end
end
-- Read input.
local luaUnprocessed, virtualPathIn
if isFile then
virtualPathIn = params.pathIn
local err
if virtualPathIn == "-" then
luaUnprocessed, err = io.stdin:read("*a")
else
luaUnprocessed, err = readFile(virtualPathIn, true)
end
if not luaUnprocessed then
errorf("Could not read file '%s'. (%s)", virtualPathIn, err)
end
current_anytime_pathIn = params.pathIn
current_anytime_pathOut = params.pathOut
else
virtualPathIn = "<code>"
luaUnprocessed = params.code
end
current_anytime_fastStrings = params.fastStrings
current_parsing_insertCount = 0
current_parsingAndMeta_resourceCache = { [virtualPathIn] = luaUnprocessed } -- The contents of files, unless params.onInsert() is specified in which case it's user defined.
current_parsingAndMeta_onInsert = params.onInsert
current_parsingAndMeta_addLineNumbers = params.addLineNumbers
current_parsingAndMeta_macroPrefix = params.macroPrefix or ""
current_parsingAndMeta_macroSuffix = params.macroSuffix or ""
current_parsingAndMeta_strictMacroArguments = params.strictMacroArguments ~= false
current_meta_locationTokens = {}
local specialFirstLine, rest = luaUnprocessed:match("^(#[^\r\n]*\r?\n?)(.*)$")
if specialFirstLine then
specialFirstLine = specialFirstLine:gsub("\r", "") -- Normalize line breaks. (Assume the input is either "\n" or "\r\n".)
luaUnprocessed = rest
end
-- Ensure there's a newline at the end of the code, otherwise there will be problems down the line.
if not (luaUnprocessed == "" or luaUnprocessed:find("\n%s*$")) then
luaUnprocessed = luaUnprocessed .. "\n"
end
local tokens = _tokenize(luaUnprocessed, virtualPathIn, true, params.backtickStrings, params.jitSyntax)
-- printTokens(tokens) -- DEBUG
-- Gather info.
local lastTok = tokens[#tokens]
local stats = {
processedByteCount = #luaUnprocessed,
lineCount = (specialFirstLine and 1 or 0)
+ (lastTok and lastTok.line + countString(lastTok.representation, "\n", true) or 0),
lineCountCode = getLineCountWithCode(tokens),
tokenCount = 0, -- Set later.
hasPreprocessorCode = false,
hasMetaprogram = false,
insertedNames = {},
}
for _, tok in ipairs(tokens) do
-- @Volatile: Make sure to update this when syntax is changed!
if isToken(tok, "pp_entry") or isToken(tok, "pp_keyword", "insert") or isToken(tok, "pp_symbol") then
stats.hasPreprocessorCode = true
stats.hasMetaprogram = true
break
elseif isToken(tok, "pp_keyword") or (isToken(tok, "string") and tok.representation:find("^`")) then
stats.hasPreprocessorCode = true
-- Keep going as there may be metaprogram.
end
end
-- Generate and run metaprogram.
----------------------------------------------------------------
local shouldProcess = stats.hasPreprocessorCode or params.addLineNumbers
if shouldProcess then
tokens = doExpansions(params, tokens, stats)
end
stats.tokenCount = #tokens
current_meta_maxLogLevel = params.logLevel or "trace"
if not LOG_LEVELS[current_meta_maxLogLevel] then
errorf(2, "Invalid 'logLevel' value in params. (%s)", tostring(current_meta_maxLogLevel))
end
local lua
if shouldProcess then
local luaMeta = astToLua(astParse(params, tokens))
--[[ DEBUG :PrintCode
print("=META===============================")
print(luaMeta)
print("====================================")
--]]
-- Run metaprogram.
current_meta_pathForErrorMessages = params.pathMeta or "<meta>"
current_meta_output = {}
current_meta_outputStack = { current_meta_output }
current_meta_canOutputNil = params.canOutputNil ~= false
current_meta_releaseMode = params.release
if params.pathMeta then
local file, err = io.open(params.pathMeta, "wb")
if not file then
errorf("Count not open '%s' for writing. (%s)", params.pathMeta, err)
end
file:write(luaMeta)
file:close()
end
if params.onBeforeMeta then
params.onBeforeMeta(luaMeta)
end
local main_chunk, err = loadLuaString(luaMeta, "@" .. current_meta_pathForErrorMessages, metaEnv)
if not main_chunk then
local ln, _err = err:match("^.-:(%d+): (.*)")
errorOnLine(current_meta_pathForErrorMessages, (tonumber(ln) or 0), nil, "%s", (_err or err))
end
current_anytime_isRunningMeta = true
main_chunk() -- Note: Our caller should clean up current_meta_pathForErrorMessages etc. on error.
current_anytime_isRunningMeta = false
if not current_parsingAndMeta_isDebug and params.pathMeta then
os.remove(params.pathMeta)
end
if current_meta_outputStack[2] then
error("Called startInterceptingOutput() more times than stopInterceptingOutput().")
end
lua = table.concat(current_meta_output)
--[[ DEBUG :PrintCode
print("=OUTPUT=============================")
print(lua)
print("====================================")
--]]
current_meta_pathForErrorMessages = ""
current_meta_output = nil
current_meta_outputStack = nil
current_meta_canOutputNil = true
current_meta_releaseMode = false
else
-- @Copypaste from above.
if not current_parsingAndMeta_isDebug and params.pathMeta then
os.remove(params.pathMeta)
end
lua = luaUnprocessed
end
current_meta_maxLogLevel = "trace"
current_meta_locationTokens = nil
if params.onAfterMeta then
local luaModified = params.onAfterMeta(lua)
if type(luaModified) == "string" then
lua = luaModified
elseif luaModified ~= nil then
errorf("onAfterMeta() did not return a string. (Got %s)", type(luaModified))
end
end
-- Write output file.
----------------------------------------------------------------
local pathOut = isFile and params.pathOut or "<output>"
if isFile then
if pathOut == "-" then
io.stdout:write(specialFirstLine or "")
io.stdout:write(lua)
else
local file, err = io.open(pathOut, "wb")
if not file then
errorf("Count not open '%s' for writing. (%s)", pathOut, err)
end
file:write(specialFirstLine or "")
file:write(lua)
file:close()
end
end
-- Check if the output is valid Lua.
if params.validate ~= false then
local luaToCheck = lua:gsub("^#![^\n]*", "")
local chunk, err = loadLuaString(luaToCheck, "@" .. pathOut, nil)
if not chunk then
local ln, _err = err:match("^.-:(%d+): (.*)")
errorOnLine(pathOut, (tonumber(ln) or 0), nil, "Output is invalid Lua. (%s)", (_err or err))
end
end
-- :ProcessInfo
local info = {
path = isFile and params.pathIn or "",
outputPath = isFile and params.pathOut or "",
processedByteCount = stats.processedByteCount,
lineCount = stats.lineCount,
linesOfCode = stats.lineCountCode,
tokenCount = stats.tokenCount,
hasPreprocessorCode = stats.hasPreprocessorCode,
hasMetaprogram = stats.hasMetaprogram,
insertedFiles = stats.insertedNames,
}
if params.onDone then
params.onDone(info)
end
current_anytime_pathIn = ""
current_anytime_pathOut = ""
current_anytime_fastStrings = false
current_parsingAndMeta_resourceCache = nil
current_parsingAndMeta_onInsert = nil
current_parsingAndMeta_addLineNumbers = false
current_parsingAndMeta_macroPrefix = ""
current_parsingAndMeta_macroSuffix = ""
current_parsingAndMeta_strictMacroArguments = true
----------------------------------------------------------------
if isFile then
return info
else
if specialFirstLine then
lua = specialFirstLine .. lua
end
return lua, info
end
end
local function processFileOrString(params, isFile)
if current_parsingAndMeta_isProcessing then
error("Cannot process recursively.", 3) -- Note: We don't return failure in this case - it's a critical error!
end
-- local startTime = os.clock() -- :DebugMeasureTime @Incomplete: Add processing time to returned info.
local returnValues = nil
current_parsingAndMeta_isProcessing = true
current_parsingAndMeta_isDebug = params.debug
local xpcallOk, xpcallErr = xpcall(function()
returnValues = pack(_processFileOrString(params, isFile))
end, function(err)
if type(err) == "string" and err:find("\0", 1, true) then
printError(tryToFormatError(cleanError(err)))
else
printErrorTraceback(err, 2) -- The level should be at error().
end
if params.onError then
local cbOk, cbErr = pcall(params.onError, err)
if not cbOk then
printfError("Additional error in params.onError()...\n%s", tryToFormatError(cbErr))
end
end
return err
end)
current_parsingAndMeta_isProcessing = false
current_parsingAndMeta_isDebug = false
-- Cleanup in case an error happened.
current_anytime_isRunningMeta = false
current_anytime_pathIn = ""
current_anytime_pathOut = ""
current_anytime_fastStrings = false
current_parsing_insertCount = 0
current_parsingAndMeta_onInsert = nil
current_parsingAndMeta_resourceCache = nil
current_parsingAndMeta_addLineNumbers = false
current_parsingAndMeta_macroPrefix = ""
current_parsingAndMeta_macroSuffix = ""
current_parsingAndMeta_strictMacroArguments = true
current_meta_pathForErrorMessages = ""
current_meta_output = nil
current_meta_outputStack = nil
current_meta_canOutputNil = true
current_meta_releaseMode = false
current_meta_maxLogLevel = "trace"
current_meta_locationTokens = nil
-- print("time", os.clock()-startTime) -- :DebugMeasureTime
if xpcallOk then
return unpack(returnValues, 1, returnValues.n)
else
return nil, cleanError(xpcallErr or "Unknown processing error.")
end
end
local function processFile(params)
local returnValues = pack(processFileOrString(params, true))
return unpack(returnValues, 1, returnValues.n)
end
local function processString(params)
local returnValues = pack(processFileOrString(params, false))
return unpack(returnValues, 1, returnValues.n)
end
-- :ExportTable
local pp = {
-- Processing functions.
----------------------------------------------------------------
-- processFile()
-- Process a Lua file. Returns nil and a message on error.
--
-- info = processFile( params )
-- info: Table with various information. (See 'ProcessInfo' for more info.)
--
-- params: Table with these fields:
-- pathIn = pathToInputFile -- [Required] Specify "-" to use stdin.
-- pathOut = pathToOutputFile -- [Required] Specify "-" to use stdout. (Note that if stdout is used then anything you print() in the metaprogram will end up there.)
-- pathMeta = pathForMetaprogram -- [Optional] You can inspect this temporary output file if an error occurs in the metaprogram.
--
-- debug = boolean -- [Optional] Debug mode. The metaprogram file is formatted more nicely and does not get deleted automatically.
-- addLineNumbers = boolean -- [Optional] Add comments with line numbers to the output.
--
-- backtickStrings = boolean -- [Optional] Enable the backtick (`) to be used as string literal delimiters. Backtick strings don't interpret any escape sequences and can't contain other backticks. (Default: false)
-- jitSyntax = boolean -- [Optional] Allow LuaJIT-specific syntax. (Default: false)
-- canOutputNil = boolean -- [Optional] Allow !(expression) and outputValue() to output nil. (Default: true)
-- fastStrings = boolean -- [Optional] Force fast serialization of string values. (Non-ASCII characters will look ugly.) (Default: false)
-- validate = boolean -- [Optional] Validate output. (Default: true)
-- strictMacroArguments = boolean -- [Optional] Check that macro arguments are valid Lua expressions. (Default: true)
--
-- macroPrefix = prefix -- [Optional] String to prepend to macro names. (Default: "")
-- macroSuffix = suffix -- [Optional] String to append to macro names. (Default: "")
--
-- release = boolean -- [Optional] Enable release mode. Currently only disables the @@ASSERT() macro when true. (Default: false)
-- logLevel = levelName -- [Optional] Maximum log level for the @@LOG() macro. Can be "off", "error", "warning", "info", "debug" or "trace". (Default: "trace", which enables all logging)
--
-- onInsert = function( name ) -- [Optional] Called for each @insert"name" instruction. It's expected to return a Lua code string. By default 'name' is a path to a file to be inserted.
-- onBeforeMeta = function( luaString ) -- [Optional] Called before the metaprogram runs, if a metaprogram is generated. luaString contains the metaprogram.
-- onAfterMeta = function( luaString ) -- [Optional] Here you can modify and return the Lua code before it's written to 'pathOut'.
-- onError = function( error ) -- [Optional] You can use this to get traceback information. 'error' is the same value as what is returned from processFile().
--
processFile = processFile,
-- processString()
-- Process Lua code. Returns nil and a message on error.
--
-- luaString, info = processString( params )
-- info: Table with various information. (See 'ProcessInfo' for more info.)
--
-- params: Table with these fields:
-- code = luaString -- [Required]
-- pathMeta = pathForMetaprogram -- [Optional] You can inspect this temporary output file if an error occurs in the metaprogram.
--
-- debug = boolean -- [Optional] Debug mode. The metaprogram file is formatted more nicely and does not get deleted automatically.
-- addLineNumbers = boolean -- [Optional] Add comments with line numbers to the output.
--
-- backtickStrings = boolean -- [Optional] Enable the backtick (`) to be used as string literal delimiters. Backtick strings don't interpret any escape sequences and can't contain other backticks. (Default: false)
-- jitSyntax = boolean -- [Optional] Allow LuaJIT-specific syntax. (Default: false)
-- canOutputNil = boolean -- [Optional] Allow !(expression) and outputValue() to output nil. (Default: true)
-- fastStrings = boolean -- [Optional] Force fast serialization of string values. (Non-ASCII characters will look ugly.) (Default: false)
-- validate = boolean -- [Optional] Validate output. (Default: true)
-- strictMacroArguments = boolean -- [Optional] Check that macro arguments are valid Lua expressions. (Default: true)
--
-- macroPrefix = prefix -- [Optional] String to prepend to macro names. (Default: "")
-- macroSuffix = suffix -- [Optional] String to append to macro names. (Default: "")
--
-- release = boolean -- [Optional] Enable release mode. Currently only disables the @@ASSERT() macro when true. (Default: false)
-- logLevel = levelName -- [Optional] Maximum log level for the @@LOG() macro. Can be "off", "error", "warning", "info", "debug" or "trace". (Default: "trace", which enables all logging)
--
-- onInsert = function( name ) -- [Optional] Called for each @insert"name" instruction. It's expected to return a Lua code string. By default 'name' is a path to a file to be inserted.
-- onBeforeMeta = function( luaString ) -- [Optional] Called before the metaprogram runs, if a metaprogram is generated. luaString contains the metaprogram.
-- onError = function( error ) -- [Optional] You can use this to get traceback information. 'error' is the same value as the second returned value from processString().
--
processString = processString,
-- Values.
----------------------------------------------------------------
VERSION = PP_VERSION, -- The version of LuaPreprocess.
metaEnvironment = metaEnv, -- The environment used for metaprograms.
}
-- Include all functions from the metaprogram environment.
for k, v in pairs(metaFuncs) do
pp[k] = v
end
return pp
--[[!===========================================================
Copyright © 2018-2022 Marcus 'ReFreezed' Thunström
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
==============================================================]]