3911 lines
125 KiB
Lua
3911 lines
125 KiB
Lua
--[[============================================================
|
|
--=
|
|
--= LuaPreprocess v1.21-dev - preprocessing library
|
|
--= by Marcus 'ReFreezed' Thunström
|
|
--=
|
|
--= License: MIT (see the bottom of this file)
|
|
--= Website: http://refreezed.com/luapreprocess/
|
|
--= Documentation: http://refreezed.com/luapreprocess/docs/
|
|
--=
|
|
--= Tested with Lua 5.1, 5.2, 5.3, 5.4 and LuaJIT.
|
|
--=
|
|
--==============================================================
|
|
|
|
API:
|
|
|
|
Global functions in metaprograms:
|
|
- copyTable
|
|
- escapePattern
|
|
- getIndentation
|
|
- isProcessing
|
|
- pack
|
|
- pairsSorted
|
|
- printf
|
|
- readFile, writeFile, fileExists
|
|
- run
|
|
- sortNatural, compareNatural
|
|
- tokenize, newToken, concatTokens, removeUselessTokens, eachToken, isToken, getNextUsefulToken
|
|
- toLua, serialize, evaluate
|
|
Only during processing:
|
|
- getCurrentPathIn, getCurrentPathOut
|
|
- getOutputSoFar, getOutputSoFarOnLine, getOutputSizeSoFar, getCurrentLineNumberInOutput, getCurrentIndentationInOutput
|
|
- loadResource, callMacro
|
|
- outputValue, outputLua, outputLuaTemplate
|
|
- startInterceptingOutput, stopInterceptingOutput
|
|
Macros:
|
|
- ASSERT
|
|
- LOG
|
|
Search this file for 'EnvironmentTable' and 'PredefinedMacros' for more info.
|
|
|
|
Exported stuff from the library:
|
|
- (all the functions above)
|
|
- VERSION
|
|
- metaEnvironment
|
|
- processFile, processString
|
|
Search this file for 'ExportTable' for more info.
|
|
|
|
----------------------------------------------------------------
|
|
|
|
How to metaprogram:
|
|
|
|
The exclamation mark (!) is used to indicate what code is part of
|
|
the metaprogram. There are 4 main ways to write metaprogram code:
|
|
|
|
!... The line will simply run during preprocessing. The line can span multiple actual lines if it contains brackets.
|
|
!!... The line will appear in both the metaprogram and the final program. The line must be an assignment.
|
|
!(...) The result of the parenthesis will be outputted as a literal if it's an expression, otherwise it'll just run.
|
|
!!(...) The result of the expression in the parenthesis will be outputted as Lua code. The result must be a string.
|
|
|
|
Short examples:
|
|
|
|
!if not isDeveloper then
|
|
sendTelemetry()
|
|
!end
|
|
|
|
!!local tau = 2*math.pi -- The expression will be evaluated in the metaprogram and the result will appear in the final program as a literal.
|
|
|
|
local bigNumber = !(5^10)
|
|
|
|
local font = !!(isDeveloper and "loadDevFont()" or "loadUserFont()")
|
|
|
|
-- See the full documentation for additional features (like macros):
|
|
-- http://refreezed.com/luapreprocess/docs/extra-functionality/
|
|
|
|
----------------------------------------------------------------
|
|
|
|
-- Example program:
|
|
|
|
-- Normal Lua.
|
|
local n = 0
|
|
doTheThing()
|
|
|
|
-- Preprocessor lines.
|
|
local n = 0
|
|
!if math.random() < 0.5 then
|
|
n = n+10 -- Normal Lua.
|
|
-- Note: In the final program, this will be in the
|
|
-- same scope as 'local n = 0' here above.
|
|
!end
|
|
|
|
!for i = 1, 3 do
|
|
print("3 lines with print().")
|
|
!end
|
|
|
|
-- Extended preprocessor line. (Lines are consumed until brackets
|
|
-- are balanced when the end of the line has been reached.)
|
|
!newClass{ -- Starts here.
|
|
name = "Entity",
|
|
props = {x=0, y=0},
|
|
} -- Ends here.
|
|
|
|
-- Preprocessor block.
|
|
!(
|
|
local dogWord = "Woof "
|
|
function getDogText()
|
|
return dogWord:rep(3)
|
|
end
|
|
)
|
|
|
|
-- Preprocessor inline block. (Expression that returns a value.)
|
|
local text = !("The dog said: "..getDogText())
|
|
|
|
-- Preprocessor inline block variant. (Expression that returns a Lua code string.)
|
|
_G.!!("myRandomGlobal"..math.random(5)) = 99
|
|
|
|
-- Dual code (both preprocessor line and final output).
|
|
!!local partial = "Hello"
|
|
local whole = partial .. !(partial..", world!")
|
|
print(whole) -- HelloHello, world!
|
|
|
|
-- Beware in preprocessor blocks that only call a single function!
|
|
!( func() ) -- This will bee seen as an inline block and output whatever value func() returns as a literal.
|
|
!( func(); ) -- If that's not wanted then a trailing `;` will prevent that. This line won't output anything by itself.
|
|
-- When the full metaprogram is generated, `!(func())` translates into `outputValue(func())`
|
|
-- while `!(func();)` simply translates into `func();` (because `outputValue(func();)` would be invalid Lua code).
|
|
-- Though in this specific case a preprocessor line (without the parenthesis) would be nicer:
|
|
!func()
|
|
|
|
-- For the full documentation, see:
|
|
-- http://refreezed.com/luapreprocess/docs/
|
|
|
|
--============================================================]]
|
|
|
|
|
|
|
|
local PP_VERSION = "1.21.0-dev"
|
|
|
|
local MAX_DUPLICATE_FILE_INSERTS = 1000 -- @Incomplete: Make this a parameter for processFile()/processString().
|
|
local MAX_CODE_LENGTH_IN_MESSAGES = 60
|
|
|
|
local KEYWORDS = {
|
|
"and","break","do","else","elseif","end","false","for","function","if","in",
|
|
"local","nil","not","or","repeat","return","then","true","until","while",
|
|
-- Lua 5.2
|
|
"goto", -- @Incomplete: A parameter to disable this for Lua 5.1?
|
|
} for i, v in ipairs(KEYWORDS) do KEYWORDS[v], KEYWORDS[i] = true, nil end
|
|
|
|
local PREPROCESSOR_KEYWORDS = {
|
|
"file","insert","line",
|
|
} for i, v in ipairs(PREPROCESSOR_KEYWORDS) do PREPROCESSOR_KEYWORDS[v], PREPROCESSOR_KEYWORDS[i] = true, nil end
|
|
|
|
local PUNCTUATION = {
|
|
"+", "-", "*", "/", "%", "^", "#",
|
|
"==", "~=", "<=", ">=", "<", ">", "=",
|
|
"(", ")", "{", "}", "[", "]",
|
|
";", ":", ",", ".", "..", "...",
|
|
-- Lua 5.2
|
|
"::",
|
|
-- Lua 5.3
|
|
"//", "&", "|", "~", ">>", "<<",
|
|
} for i, v in ipairs(PUNCTUATION) do PUNCTUATION[v], PUNCTUATION[i] = true, nil end
|
|
|
|
local ESCAPE_SEQUENCES_EXCEPT_QUOTES = {
|
|
["\a"] = [[\a]],
|
|
["\b"] = [[\b]],
|
|
["\f"] = [[\f]],
|
|
["\n"] = [[\n]],
|
|
["\r"] = [[\r]],
|
|
["\t"] = [[\t]],
|
|
["\v"] = [[\v]],
|
|
["\\"] = [[\\]],
|
|
}
|
|
local ESCAPE_SEQUENCES = {
|
|
["\""] = [[\"]],
|
|
["\'"] = [[\']],
|
|
} for k, v in pairs(ESCAPE_SEQUENCES_EXCEPT_QUOTES) do ESCAPE_SEQUENCES[k] = v end
|
|
|
|
local USELESS_TOKENS = {whitespace=true, comment=true}
|
|
|
|
local LOG_LEVELS = {
|
|
["off" ] = 0,
|
|
["error" ] = 1,
|
|
["warning"] = 2,
|
|
["info" ] = 3,
|
|
["debug" ] = 4,
|
|
["trace" ] = 5,
|
|
}
|
|
|
|
local metaEnv = nil
|
|
local dummyEnv = {}
|
|
|
|
-- Controlled by processFileOrString():
|
|
local current_parsingAndMeta_isProcessing = false
|
|
local current_parsingAndMeta_isDebug = false
|
|
|
|
-- Controlled by _processFileOrString():
|
|
local current_anytime_isRunningMeta = false
|
|
local current_anytime_pathIn = ""
|
|
local current_anytime_pathOut = ""
|
|
local current_anytime_fastStrings = false
|
|
local current_parsing_insertCount = 0
|
|
local current_parsingAndMeta_onInsert = nil
|
|
local current_parsingAndMeta_resourceCache = nil
|
|
local current_parsingAndMeta_addLineNumbers = false
|
|
local current_parsingAndMeta_macroPrefix = ""
|
|
local current_parsingAndMeta_macroSuffix = ""
|
|
local current_parsingAndMeta_strictMacroArguments = true
|
|
local current_meta_pathForErrorMessages = ""
|
|
local current_meta_output = nil -- Top item in current_meta_outputStack.
|
|
local current_meta_outputStack = nil
|
|
local current_meta_canOutputNil = true
|
|
local current_meta_releaseMode = false
|
|
local current_meta_maxLogLevel = "trace"
|
|
local current_meta_locationTokens = nil
|
|
|
|
|
|
|
|
--==============================================================
|
|
--= Local Functions ============================================
|
|
--==============================================================
|
|
|
|
local assertarg
|
|
local countString, countSubString
|
|
local getLineNumber
|
|
local loadLuaString
|
|
local maybeOutputLineNumber
|
|
local sortNatural
|
|
local tableInsert, tableRemove, tableInsertFormat
|
|
local utf8GetCodepointAndLength
|
|
|
|
|
|
|
|
local F = string.format
|
|
|
|
local function tryToFormatError(err0)
|
|
local err, path, ln = nil
|
|
|
|
if type(err0) == "string" then
|
|
do path, ln, err = err0:match"^(%a:[%w_/\\.]+):(%d+): (.*)"
|
|
if not err then path, ln, err = err0:match"^([%w_/\\.]+):(%d+): (.*)"
|
|
if not err then path, ln, err = err0:match"^(%S-):(%d+): (.*)"
|
|
end end end
|
|
end
|
|
|
|
if err then
|
|
return F("Error @ %s:%s: %s", path, ln, err)
|
|
else
|
|
return "Error: "..tostring(err0)
|
|
end
|
|
end
|
|
|
|
|
|
|
|
local function printf(s, ...)
|
|
print(F(s, ...))
|
|
end
|
|
|
|
-- printTokens( tokens [, filterUselessTokens ] )
|
|
local function printTokens(tokens, filter)
|
|
for i, tok in ipairs(tokens) do
|
|
if not (filter and USELESS_TOKENS[tok.type]) then
|
|
printf("%d %-12s '%s'", i, tok.type, (F("%q", tostring(tok.value)):sub(2, -2):gsub("\\\n", "\\n")))
|
|
end
|
|
end
|
|
end
|
|
|
|
local function printError(s)
|
|
io.stderr:write(s, "\n")
|
|
end
|
|
local function printfError(s, ...)
|
|
printError(F(s, ...))
|
|
end
|
|
|
|
-- message = formatTraceback( [ level=1 ] )
|
|
local function formatTraceback(level)
|
|
local buffer = {}
|
|
tableInsert(buffer, "stack traceback:\n")
|
|
|
|
level = 1 + (level or 1)
|
|
local stack = {}
|
|
|
|
while level < 1/0 do
|
|
local info = debug.getinfo(level, "nSl")
|
|
if not info then break end
|
|
|
|
local isFile = info.source:find"^@" ~= nil
|
|
local sourceName = (isFile and info.source:sub(2) or info.short_src)
|
|
|
|
local subBuffer = {"\t"}
|
|
tableInsertFormat(subBuffer, "%s:", sourceName)
|
|
|
|
if info.currentline > 0 then
|
|
tableInsertFormat(subBuffer, "%d:", info.currentline)
|
|
end
|
|
|
|
if (info.name or "") ~= "" then
|
|
tableInsertFormat(subBuffer, " in '%s'", info.name)
|
|
elseif info.what == "main" then
|
|
tableInsert(subBuffer, " in main chunk")
|
|
elseif info.what == "C" or info.what == "tail" then
|
|
tableInsert(subBuffer, " ?")
|
|
else
|
|
tableInsertFormat(subBuffer, " in <%s:%d>", sourceName:gsub("^.*[/\\]", ""), info.linedefined)
|
|
end
|
|
|
|
tableInsert(stack, table.concat(subBuffer))
|
|
level = level + 1
|
|
end
|
|
|
|
while stack[#stack] == "\t[C]: ?" do
|
|
stack[#stack] = nil
|
|
end
|
|
|
|
for _, s in ipairs(stack) do
|
|
tableInsert(buffer, s)
|
|
tableInsert(buffer, "\n")
|
|
end
|
|
|
|
return table.concat(buffer)
|
|
end
|
|
|
|
-- printErrorTraceback( message [, level=1 ] )
|
|
local function printErrorTraceback(message, level)
|
|
printError(tryToFormatError(message))
|
|
printError(formatTraceback(1+(level or 1)))
|
|
end
|
|
|
|
-- debugExit( )
|
|
-- debugExit( messageValue )
|
|
-- debugExit( messageFormat, ... )
|
|
local function debugExit(...)
|
|
if select("#", ...) > 1 then
|
|
printfError(...)
|
|
elseif select("#", ...) == 1 then
|
|
printError(...)
|
|
end
|
|
os.exit(2)
|
|
end
|
|
|
|
|
|
|
|
-- errorf( [ level=1, ] string, ... )
|
|
local function errorf(sOrLevel, ...)
|
|
if type(sOrLevel) == "number" then
|
|
error(F(...), (sOrLevel == 0 and 0 or 1+sOrLevel))
|
|
else
|
|
error(F(sOrLevel, ...), 2)
|
|
end
|
|
end
|
|
|
|
-- local function errorLine(err) -- Unused.
|
|
-- if type(err) ~= "string" then error(err) end
|
|
-- error("\0"..err, 0) -- The 0 tells our own error handler not to print the traceback.
|
|
-- end
|
|
local function errorfLine(s, ...)
|
|
errorf(0, (current_parsingAndMeta_isProcessing and "\0" or "")..s, ...) -- The \0 tells our own error handler not to print the traceback.
|
|
end
|
|
|
|
-- errorOnLine( path, lineNumber, agent=nil, s, ... )
|
|
local function errorOnLine(path, ln, agent, s, ...)
|
|
s = F(s, ...)
|
|
if agent then
|
|
errorfLine("%s:%d: [%s] %s", path, ln, agent, s)
|
|
else
|
|
errorfLine("%s:%d: %s", path, ln, s)
|
|
end
|
|
end
|
|
|
|
local errorInFile, runtimeErrorInFile
|
|
do
|
|
local function findStartOfLine(s, pos, canBeEmpty)
|
|
while pos > 1 do
|
|
if s:byte(pos-1) == 10--[[\n]] and (canBeEmpty or s:byte(pos) ~= 10--[[\n]]) then break end
|
|
pos = pos - 1
|
|
end
|
|
return math.max(pos, 1)
|
|
end
|
|
local function findEndOfLine(s, pos)
|
|
while pos < #s do
|
|
if s:byte(pos+1) == 10--[[\n]] then break end
|
|
pos = pos + 1
|
|
end
|
|
return math.min(pos, #s)
|
|
end
|
|
|
|
local function _errorInFile(level, contents, path, pos, agent, s, ...)
|
|
s = F(s, ...)
|
|
|
|
pos = math.min(math.max(pos, 1), #contents+1)
|
|
local ln = getLineNumber(contents, pos)
|
|
|
|
local lineStart = findStartOfLine(contents, pos, true)
|
|
local lineEnd = findEndOfLine (contents, pos-1)
|
|
local linePre1Start = findStartOfLine(contents, lineStart-1, false)
|
|
local linePre1End = findEndOfLine (contents, linePre1Start-1)
|
|
local linePre2Start = findStartOfLine(contents, linePre1Start-1, false)
|
|
local linePre2End = findEndOfLine (contents, linePre2Start-1)
|
|
-- printfError("pos %d | lines %d..%d, %d..%d, %d..%d", pos, linePre2Start,linePre2End+1, linePre1Start,linePre1End+1, lineStart,lineEnd+1) -- DEBUG
|
|
|
|
errorOnLine(path, ln, agent, "%s\n>\n%s%s%s>-%s^%s",
|
|
s,
|
|
(linePre2Start < linePre1Start and linePre2Start <= linePre2End) and F("> %s\n", (contents:sub(linePre2Start, linePre2End):gsub("\t", " "))) or "",
|
|
(linePre1Start < lineStart and linePre1Start <= linePre1End) and F("> %s\n", (contents:sub(linePre1Start, linePre1End):gsub("\t", " "))) or "",
|
|
( lineStart <= lineEnd ) and F("> %s\n", (contents:sub(lineStart, lineEnd ):gsub("\t", " "))) or ">\n",
|
|
("-"):rep(pos - lineStart + 3*countSubString(contents, lineStart, lineEnd, "\t", true)),
|
|
(level and "\n"..formatTraceback(1+level) or "")
|
|
)
|
|
end
|
|
|
|
-- errorInFile( contents, path, pos, agent, s, ... )
|
|
--[[local]] function errorInFile(...)
|
|
_errorInFile(nil, ...)
|
|
end
|
|
|
|
-- runtimeErrorInFile( level, contents, path, pos, agent, s, ... )
|
|
--[[local]] function runtimeErrorInFile(level, ...)
|
|
_errorInFile(1+level, ...)
|
|
end
|
|
end
|
|
|
|
-- errorAtToken( token, position=token.position, agent, s, ... )
|
|
local function errorAtToken(tok, pos, agent, s, ...)
|
|
-- printErrorTraceback("errorAtToken", 2) -- DEBUG
|
|
errorInFile(current_parsingAndMeta_resourceCache[tok.file], tok.file, (pos or tok.position), agent, s, ...)
|
|
end
|
|
|
|
-- errorAfterToken( token, agent, s, ... )
|
|
local function errorAfterToken(tok, agent, s, ...)
|
|
-- printErrorTraceback("errorAfterToken", 2) -- DEBUG
|
|
errorInFile(current_parsingAndMeta_resourceCache[tok.file], tok.file, tok.position+#tok.representation, agent, s, ...)
|
|
end
|
|
|
|
-- runtimeErrorAtToken( level, token, position=token.position, agent, s, ... )
|
|
local function runtimeErrorAtToken(level, tok, pos, agent, s, ...)
|
|
-- printErrorTraceback("runtimeErrorAtToken", 2) -- DEBUG
|
|
runtimeErrorInFile(1+level, current_parsingAndMeta_resourceCache[tok.file], tok.file, (pos or tok.position), agent, s, ...)
|
|
end
|
|
|
|
-- internalError( [ message|value ] )
|
|
local function internalError(message)
|
|
message = message and " ("..tostring(message)..")" or ""
|
|
error("Internal error."..message, 2)
|
|
end
|
|
|
|
|
|
|
|
local function cleanError(err)
|
|
if type(err) == "string" then
|
|
err = err:gsub("%z", "")
|
|
end
|
|
return err
|
|
end
|
|
|
|
|
|
|
|
local function formatCodeForShortMessage(lua)
|
|
lua = lua:gsub("^%s+", ""):gsub("%s+$", ""):gsub("%s+", " ")
|
|
|
|
if #lua > MAX_CODE_LENGTH_IN_MESSAGES then
|
|
lua = lua:sub(1, MAX_CODE_LENGTH_IN_MESSAGES/2) .. "..." .. lua:sub(-MAX_CODE_LENGTH_IN_MESSAGES/2)
|
|
end
|
|
|
|
return lua
|
|
end
|
|
|
|
|
|
|
|
local ERROR_UNFINISHED_STRINGLIKE = 1
|
|
|
|
local function parseStringlikeToken(s, ptr)
|
|
local reprStart = ptr
|
|
local reprEnd
|
|
|
|
local valueStart
|
|
local valueEnd
|
|
|
|
local longEqualSigns = s:match("^%[(=*)%[", ptr)
|
|
local isLong = longEqualSigns ~= nil
|
|
|
|
-- Single line.
|
|
if not isLong then
|
|
valueStart = ptr
|
|
|
|
local i = s:find("\n", ptr, true)
|
|
if not i then
|
|
reprEnd = #s
|
|
valueEnd = #s
|
|
ptr = reprEnd + 1
|
|
else
|
|
reprEnd = i
|
|
valueEnd = i - 1
|
|
ptr = reprEnd + 1
|
|
end
|
|
|
|
-- Multiline.
|
|
else
|
|
ptr = ptr + 1 + #longEqualSigns + 1
|
|
valueStart = ptr
|
|
|
|
local i1, i2 = s:find("]"..longEqualSigns.."]", ptr, true)
|
|
if not i1 then
|
|
return nil, ERROR_UNFINISHED_STRINGLIKE
|
|
end
|
|
|
|
reprEnd = i2
|
|
valueEnd = i1 - 1
|
|
ptr = reprEnd + 1
|
|
end
|
|
|
|
local repr = s:sub(reprStart, reprEnd)
|
|
local v = s:sub(valueStart, valueEnd)
|
|
local tok = {type="stringlike", representation=repr, value=v, long=isLong}
|
|
|
|
return tok, ptr
|
|
end
|
|
|
|
|
|
|
|
local NUM_HEX_FRAC_EXP = ("^( 0[Xx] (%x*) %.(%x+) [Pp]([-+]?%x+) )"):gsub(" +", "")
|
|
local NUM_HEX_FRAC = ("^( 0[Xx] (%x*) %.(%x+) )"):gsub(" +", "")
|
|
local NUM_HEX_EXP = ("^( 0[Xx] (%x+) %.? [Pp]([-+]?%x+) )"):gsub(" +", "")
|
|
local NUM_HEX = ("^( 0[Xx] %x+ %.? )"):gsub(" +", "")
|
|
local NUM_DEC_FRAC_EXP = ("^( %d* %.%d+ [Ee][-+]?%d+ )"):gsub(" +", "")
|
|
local NUM_DEC_FRAC = ("^( %d* %.%d+ )"):gsub(" +", "")
|
|
local NUM_DEC_EXP = ("^( %d+ %.? [Ee][-+]?%d+ )"):gsub(" +", "")
|
|
local NUM_DEC = ("^( %d+ %.? )"):gsub(" +", "")
|
|
|
|
-- tokens = _tokenize( luaString, path, allowPreprocessorTokens, allowBacktickStrings, allowJitSyntax )
|
|
local function _tokenize(s, path, allowPpTokens, allowBacktickStrings, allowJitSyntax)
|
|
s = s:gsub("\r", "") -- Normalize line breaks. (Assume the input is either "\n" or "\r\n".)
|
|
|
|
local tokens = {}
|
|
local ptr = 1
|
|
local ln = 1
|
|
|
|
while ptr <= #s do
|
|
local tok
|
|
local tokenPos = ptr
|
|
|
|
-- Whitespace.
|
|
if s:find("^%s", ptr) then
|
|
local i1, i2, whitespace = s:find("^(%s+)", ptr)
|
|
|
|
ptr = i2+1
|
|
tok = {type="whitespace", representation=whitespace, value=whitespace}
|
|
|
|
-- Identifier/keyword.
|
|
elseif s:find("^[%a_]", ptr) then
|
|
local i1, i2, word = s:find("^([%a_][%w_]*)", ptr)
|
|
ptr = i2+1
|
|
|
|
if KEYWORDS[word] then
|
|
tok = {type="keyword", representation=word, value=word}
|
|
else
|
|
tok = {type="identifier", representation=word, value=word}
|
|
end
|
|
|
|
-- Number (binary).
|
|
elseif s:find("^0b", ptr) then
|
|
if not allowJitSyntax then
|
|
errorInFile(s, path, ptr, "Tokenizer", "Encountered binary numeral. (Feature not enabled.)")
|
|
end
|
|
|
|
local i1, i2, numStr = s:find("^(..[01]+)", ptr)
|
|
|
|
-- @Copypaste from below.
|
|
if not numStr then
|
|
errorInFile(s, path, ptr, "Tokenizer", "Malformed number.")
|
|
end
|
|
|
|
local numStrFallback = numStr
|
|
|
|
do
|
|
if s:find("^[Ii]", i2+1) then -- Imaginary part of complex number.
|
|
numStr = s:sub(i1, i2+1)
|
|
i2 = i2 + 1
|
|
|
|
elseif s:find("^[Uu][Ll][Ll]", i2+1) then -- Unsigned 64-bit integer.
|
|
numStr = s:sub(i1, i2+3)
|
|
i2 = i2 + 3
|
|
elseif s:find("^[Ll][Ll]", i2+1) then -- Signed 64-bit integer.
|
|
numStr = s:sub(i1, i2+2)
|
|
i2 = i2 + 2
|
|
end
|
|
end
|
|
|
|
local n = tonumber(numStr) or tonumber(numStrFallback) or tonumber(numStrFallback:sub(3), 2)
|
|
|
|
if not n then
|
|
errorInFile(s, path, ptr, "Tokenizer", "Invalid number.")
|
|
end
|
|
|
|
if s:find("^[%w_]", i2+1) then
|
|
-- This is actually not an error in Lua 5.2 and 5.3. Maybe we should issue a warning instead of an error here?
|
|
errorInFile(s, path, i2+1, "Tokenizer", "Malformed number.")
|
|
end
|
|
|
|
ptr = i2 + 1
|
|
tok = {type="number", representation=numStrFallback, value=n}
|
|
|
|
-- Number.
|
|
elseif s:find("^%.?%d", ptr) then
|
|
local pat, maybeInt, lua52Hex, i1, i2, numStr = NUM_HEX_FRAC_EXP, false, true , s:find(NUM_HEX_FRAC_EXP, ptr)
|
|
if not i1 then pat, maybeInt, lua52Hex, i1, i2, numStr = NUM_HEX_FRAC , false, true , s:find(NUM_HEX_FRAC , ptr)
|
|
if not i1 then pat, maybeInt, lua52Hex, i1, i2, numStr = NUM_HEX_EXP , false, true , s:find(NUM_HEX_EXP , ptr)
|
|
if not i1 then pat, maybeInt, lua52Hex, i1, i2, numStr = NUM_HEX , true , false, s:find(NUM_HEX , ptr)
|
|
if not i1 then pat, maybeInt, lua52Hex, i1, i2, numStr = NUM_DEC_FRAC_EXP, false, false, s:find(NUM_DEC_FRAC_EXP, ptr)
|
|
if not i1 then pat, maybeInt, lua52Hex, i1, i2, numStr = NUM_DEC_FRAC , false, false, s:find(NUM_DEC_FRAC , ptr)
|
|
if not i1 then pat, maybeInt, lua52Hex, i1, i2, numStr = NUM_DEC_EXP , false, false, s:find(NUM_DEC_EXP , ptr)
|
|
if not i1 then pat, maybeInt, lua52Hex, i1, i2, numStr = NUM_DEC , true , false, s:find(NUM_DEC , ptr)
|
|
end end end end end end end
|
|
|
|
if not numStr then
|
|
errorInFile(s, path, ptr, "Tokenizer", "Malformed number.")
|
|
end
|
|
|
|
local numStrFallback = numStr
|
|
|
|
if allowJitSyntax then
|
|
if s:find("^[Ii]", i2+1) then -- Imaginary part of complex number.
|
|
numStr = s:sub(i1, i2+1)
|
|
i2 = i2 + 1
|
|
|
|
elseif not maybeInt or numStr:find(".", 1, true) then
|
|
-- void
|
|
|
|
elseif s:find("^[Uu][Ll][Ll]", i2+1) then -- Unsigned 64-bit integer.
|
|
numStr = s:sub(i1, i2+3)
|
|
i2 = i2 + 3
|
|
elseif s:find("^[Ll][Ll]", i2+1) then -- Signed 64-bit integer.
|
|
numStr = s:sub(i1, i2+2)
|
|
i2 = i2 + 2
|
|
end
|
|
end
|
|
|
|
local n = tonumber(numStr) or tonumber(numStrFallback)
|
|
|
|
-- Support hexadecimal floats in Lua 5.1.
|
|
if not n and lua52Hex then
|
|
-- Note: We know we're not running LuaJIT here as it supports hexadecimal floats, thus we use numStrFallback instead of numStr.
|
|
local _, intStr, fracStr, expStr
|
|
if pat == NUM_HEX_FRAC_EXP then _, intStr, fracStr, expStr = numStrFallback:match(NUM_HEX_FRAC_EXP)
|
|
elseif pat == NUM_HEX_FRAC then _, intStr, fracStr = numStrFallback:match(NUM_HEX_FRAC) ; expStr = "0"
|
|
elseif pat == NUM_HEX_EXP then _, intStr, expStr = numStrFallback:match(NUM_HEX_EXP) ; fracStr = ""
|
|
else internalError() end
|
|
|
|
n = tonumber(intStr, 16) or 0 -- intStr may be "".
|
|
|
|
local fracValue = 1
|
|
for i = 1, #fracStr do
|
|
fracValue = fracValue/16
|
|
n = n+tonumber(fracStr:sub(i, i), 16)*fracValue
|
|
end
|
|
|
|
n = n*2^expStr:gsub("^+", "")
|
|
end
|
|
|
|
if not n then
|
|
errorInFile(s, path, ptr, "Tokenizer", "Invalid number.")
|
|
end
|
|
|
|
if s:find("^[%w_]", i2+1) then
|
|
-- This is actually not an error in Lua 5.2 and 5.3. Maybe we should issue a warning instead of an error here?
|
|
errorInFile(s, path, i2+1, "Tokenizer", "Malformed number.")
|
|
end
|
|
|
|
ptr = i2+1
|
|
tok = {type="number", representation=numStrFallback, value=n}
|
|
|
|
-- Comment.
|
|
elseif s:find("^%-%-", ptr) then
|
|
local reprStart = ptr
|
|
ptr = ptr+2
|
|
|
|
tok, ptr = parseStringlikeToken(s, ptr)
|
|
if not tok then
|
|
local errCode = ptr
|
|
if errCode == ERROR_UNFINISHED_STRINGLIKE then
|
|
errorInFile(s, path, reprStart, "Tokenizer", "Unfinished long comment.")
|
|
else
|
|
errorInFile(s, path, reprStart, "Tokenizer", "Invalid comment.")
|
|
end
|
|
end
|
|
|
|
if tok.long then
|
|
-- Check for nesting of [[...]], which is deprecated in Lua.
|
|
local chunk, err = loadLuaString("--"..tok.representation, "@", nil)
|
|
|
|
if not chunk then
|
|
local lnInString, luaErr = err:match'^:(%d+): (.*)'
|
|
if luaErr then
|
|
errorOnLine(path, getLineNumber(s, reprStart)+tonumber(lnInString)-1, "Tokenizer", "Malformed long comment. (%s)", luaErr)
|
|
else
|
|
errorInFile(s, path, reprStart, "Tokenizer", "Malformed long comment.")
|
|
end
|
|
end
|
|
end
|
|
|
|
tok.type = "comment"
|
|
tok.representation = s:sub(reprStart, ptr-1)
|
|
|
|
-- String (short).
|
|
elseif s:find([=[^["']]=], ptr) then
|
|
local reprStart = ptr
|
|
local reprEnd
|
|
|
|
local quoteChar = s:sub(ptr, ptr)
|
|
ptr = ptr+1
|
|
|
|
local valueStart = ptr
|
|
local valueEnd
|
|
|
|
while true do
|
|
local c = s:sub(ptr, ptr)
|
|
|
|
if c == "" then
|
|
errorInFile(s, path, reprStart, "Tokenizer", "Unfinished string.")
|
|
|
|
elseif c == quoteChar then
|
|
reprEnd = ptr
|
|
valueEnd = ptr-1
|
|
ptr = reprEnd+1
|
|
break
|
|
|
|
elseif c == "\\" then
|
|
-- Note: We don't have to look for multiple characters after
|
|
-- the escape, like \nnn - this algorithm works anyway.
|
|
if ptr+1 > #s then
|
|
errorInFile(s, path, reprStart, "Tokenizer", "Unfinished string after escape.")
|
|
end
|
|
ptr = ptr+2
|
|
|
|
elseif c == "\n" then
|
|
-- Can't have unescaped newlines. Lua, this is a silly rule! @Ugh
|
|
errorInFile(s, path, ptr, "Tokenizer", "Newlines must be escaped in strings.")
|
|
|
|
else
|
|
ptr = ptr+1
|
|
end
|
|
end
|
|
|
|
local repr = s:sub(reprStart, reprEnd)
|
|
|
|
local valueChunk = loadLuaString("return"..repr, nil, nil)
|
|
if not valueChunk then
|
|
errorInFile(s, path, reprStart, "Tokenizer", "Malformed string.")
|
|
end
|
|
|
|
local v = valueChunk()
|
|
assert(type(v) == "string")
|
|
|
|
tok = {type="string", representation=repr, value=valueChunk(), long=false}
|
|
|
|
-- Long string.
|
|
elseif s:find("^%[=*%[", ptr) then
|
|
local reprStart = ptr
|
|
|
|
tok, ptr = parseStringlikeToken(s, ptr)
|
|
if not tok then
|
|
local errCode = ptr
|
|
if errCode == ERROR_UNFINISHED_STRINGLIKE then
|
|
errorInFile(s, path, reprStart, "Tokenizer", "Unfinished long string.")
|
|
else
|
|
errorInFile(s, path, reprStart, "Tokenizer", "Invalid long string.")
|
|
end
|
|
end
|
|
|
|
-- Check for nesting of [[...]], which is deprecated in Lua.
|
|
local valueChunk, err = loadLuaString("return"..tok.representation, "@", nil)
|
|
|
|
if not valueChunk then
|
|
local lnInString, luaErr = err:match'^:(%d+): (.*)'
|
|
if luaErr then
|
|
errorOnLine(path, getLineNumber(s, reprStart)+tonumber(lnInString)-1, "Tokenizer", "Malformed long string. (%s)", luaErr)
|
|
else
|
|
errorInFile(s, path, reprStart, "Tokenizer", "Malformed long string.")
|
|
end
|
|
end
|
|
|
|
local v = valueChunk()
|
|
assert(type(v) == "string")
|
|
|
|
tok.type = "string"
|
|
tok.value = v
|
|
|
|
-- Backtick string.
|
|
elseif s:find("^`", ptr) then
|
|
if not allowBacktickStrings then
|
|
errorInFile(s, path, ptr, "Tokenizer", "Encountered backtick string. (Feature not enabled.)")
|
|
end
|
|
|
|
local i1, i2, repr, v = s:find("^(`([^`]*)`)", ptr)
|
|
if not i2 then
|
|
errorInFile(s, path, ptr, "Tokenizer", "Unfinished backtick string.")
|
|
end
|
|
|
|
ptr = i2+1
|
|
tok = {type="string", representation=repr, value=v, long=false}
|
|
|
|
-- Punctuation etc.
|
|
elseif s:find("^%.%.%.", ptr) then -- 3
|
|
local repr = s:sub(ptr, ptr+2)
|
|
tok = {type="punctuation", representation=repr, value=repr}
|
|
ptr = ptr+#repr
|
|
elseif s:find("^%.%.", ptr) or s:find("^[=~<>]=", ptr) or s:find("^::", ptr) or s:find("^//", ptr) or s:find("^<<", ptr) or s:find("^>>", ptr) then -- 2
|
|
local repr = s:sub(ptr, ptr+1)
|
|
tok = {type="punctuation", representation=repr, value=repr}
|
|
ptr = ptr+#repr
|
|
elseif s:find("^[+%-*/%%^#<>=(){}[%];:,.&|~]", ptr) then -- 1
|
|
local repr = s:sub(ptr, ptr)
|
|
tok = {type="punctuation", representation=repr, value=repr}
|
|
ptr = ptr+#repr
|
|
|
|
-- Preprocessor entry.
|
|
elseif s:find("^!", ptr) then
|
|
if not allowPpTokens then
|
|
errorInFile(s, path, ptr, "Tokenizer", "Encountered preprocessor entry. (Feature not enabled.)")
|
|
end
|
|
|
|
local double = s:find("^!", ptr+1) ~= nil
|
|
local repr = s:sub(ptr, ptr+(double and 1 or 0))
|
|
|
|
tok = {type="pp_entry", representation=repr, value=repr, double=double}
|
|
ptr = ptr+#repr
|
|
|
|
-- Preprocessor keyword.
|
|
elseif s:find("^@", ptr) then
|
|
if not allowPpTokens then
|
|
errorInFile(s, path, ptr, "Tokenizer", "Encountered preprocessor keyword. (Feature not enabled.)")
|
|
end
|
|
|
|
if s:find("^@@", ptr) then
|
|
ptr = ptr+2
|
|
tok = {type="pp_keyword", representation="@@", value="insert"}
|
|
else
|
|
local i1, i2, repr, word = s:find("^(@([%a_][%w_]*))", ptr)
|
|
if not i1 then
|
|
errorInFile(s, path, ptr+1, "Tokenizer", "Expected an identifier.")
|
|
elseif not PREPROCESSOR_KEYWORDS[word] then
|
|
errorInFile(s, path, ptr+1, "Tokenizer", "Invalid preprocessor keyword '%s'.", word)
|
|
end
|
|
ptr = i2+1
|
|
tok = {type="pp_keyword", representation=repr, value=word}
|
|
end
|
|
|
|
-- Preprocessor symbol.
|
|
elseif s:find("^%$", ptr) then
|
|
if not allowPpTokens then
|
|
errorInFile(s, path, ptr, "Tokenizer", "Encountered preprocessor symbol. (Feature not enabled.)")
|
|
end
|
|
|
|
local i1, i2, repr, word = s:find("^(%$([%a_][%w_]*))", ptr)
|
|
if not i1 then
|
|
errorInFile(s, path, ptr+1, "Tokenizer", "Expected an identifier.")
|
|
elseif KEYWORDS[word] then
|
|
errorInFile(s, path, ptr+1, "Tokenizer", "Invalid preprocessor symbol '%s'. (Must not be a Lua keyword.)", word)
|
|
end
|
|
ptr = i2+1
|
|
tok = {type="pp_symbol", representation=repr, value=word}
|
|
|
|
else
|
|
errorInFile(s, path, ptr, "Tokenizer", "Unknown character.")
|
|
end
|
|
|
|
tok.line = ln
|
|
tok.position = tokenPos
|
|
tok.file = path
|
|
|
|
ln = ln+countString(tok.representation, "\n", true)
|
|
tok.lineEnd = ln
|
|
|
|
tableInsert(tokens, tok)
|
|
-- print(#tokens, tok.type, tok.representation) -- DEBUG
|
|
end
|
|
|
|
return tokens
|
|
end
|
|
|
|
|
|
|
|
-- luaString = _concatTokens( tokens, lastLn=nil, addLineNumbers, fromIndex=1, toIndex=#tokens )
|
|
local function _concatTokens(tokens, lastLn, addLineNumbers, i1, i2)
|
|
local parts = {}
|
|
|
|
if addLineNumbers then
|
|
for i = (i1 or 1), (i2 or #tokens) do
|
|
local tok = tokens[i]
|
|
lastLn = maybeOutputLineNumber(parts, tok, lastLn)
|
|
tableInsert(parts, tok.representation)
|
|
end
|
|
|
|
else
|
|
for i = (i1 or 1), (i2 or #tokens) do
|
|
tableInsert(parts, tokens[i].representation)
|
|
end
|
|
end
|
|
|
|
return table.concat(parts)
|
|
end
|
|
|
|
local function insertTokenRepresentations(parts, tokens, i1, i2)
|
|
for i = i1, i2 do
|
|
tableInsert(parts, tokens[i].representation)
|
|
end
|
|
end
|
|
|
|
|
|
|
|
local function readFile(path, isTextFile)
|
|
assertarg(1, path, "string")
|
|
assertarg(2, isTextFile, "boolean","nil")
|
|
|
|
local file, err = io.open(path, "r"..(isTextFile and "" or "b"))
|
|
if not file then return nil, err end
|
|
|
|
local contents = file:read"*a"
|
|
file:close()
|
|
return contents
|
|
end
|
|
|
|
-- success, error = writeFile( path, [ isTextFile=false, ] contents )
|
|
local function writeFile(path, isTextFile, contents)
|
|
assertarg(1, path, "string")
|
|
|
|
if type(isTextFile) == "boolean" then
|
|
assertarg(3, contents, "string")
|
|
else
|
|
isTextFile, contents = false, isTextFile
|
|
assertarg(2, contents, "string")
|
|
end
|
|
|
|
local file, err = io.open(path, "w"..(isTextFile and "" or "b"))
|
|
if not file then return false, err end
|
|
|
|
file:write(contents)
|
|
file:close()
|
|
return true
|
|
end
|
|
|
|
local function fileExists(path)
|
|
assertarg(1, path, "string")
|
|
|
|
local file = io.open(path, "r")
|
|
if not file then return false end
|
|
|
|
file:close()
|
|
return true
|
|
end
|
|
|
|
|
|
|
|
-- assertarg( argumentNumber, value, expectedValueType1, ... )
|
|
--[[local]] function assertarg(n, v, ...)
|
|
local vType = type(v)
|
|
|
|
for i = 1, select("#", ...) do
|
|
if vType == select(i, ...) then return end
|
|
end
|
|
|
|
local fName = debug.getinfo(2, "n").name
|
|
local expects = table.concat({...}, " or ")
|
|
|
|
if fName == "" then fName = "?" end
|
|
|
|
errorf(3, "bad argument #%d to '%s' (%s expected, got %s)", n, fName, expects, vType)
|
|
end
|
|
|
|
|
|
|
|
-- count = countString( haystack, needle [, plain=false ] )
|
|
--[[local]] function countString(s, needle, plain)
|
|
local count = 0
|
|
local i = 0
|
|
local _
|
|
|
|
while true do
|
|
_, i = s:find(needle, i+1, plain)
|
|
if not i then return count end
|
|
|
|
count = count+1
|
|
end
|
|
end
|
|
|
|
-- count = countSubString( string, startPosition, endPosition, needle [, plain=false ] )
|
|
--[[local]] function countSubString(s, pos, posEnd, needle, plain)
|
|
local count = 0
|
|
|
|
while true do
|
|
local _, i2 = s:find(needle, pos, plain)
|
|
if not i2 or i2 > posEnd then return count end
|
|
|
|
count = count + 1
|
|
pos = i2 + 1
|
|
end
|
|
end
|
|
|
|
|
|
|
|
local getfenv = getfenv or function(f) -- Assume Lua is version 5.2+ if getfenv() doesn't exist.
|
|
f = f or 1
|
|
|
|
if type(f) == "function" then
|
|
-- void
|
|
|
|
elseif type(f) == "number" then
|
|
if f == 0 then return _ENV end
|
|
if f < 0 then error("bad argument #1 to 'getfenv' (level must be non-negative)") end
|
|
|
|
f = debug.getinfo(1+f, "f") or error("bad argument #1 to 'getfenv' (invalid level)")
|
|
f = f.func
|
|
|
|
else
|
|
error("bad argument #1 to 'getfenv' (number expected, got "..type(f)..")")
|
|
end
|
|
|
|
for i = 1, 1/0 do
|
|
local name, v = debug.getupvalue(f, i)
|
|
if name == "_ENV" then return v end
|
|
if not name then return _ENV end
|
|
end
|
|
end
|
|
|
|
|
|
|
|
-- (Table generated by misc/generateStringEscapeSequenceInfo.lua)
|
|
local UNICODE_RANGES_NOT_TO_ESCAPE = {
|
|
{from=32, to=126},
|
|
{from=161, to=591},
|
|
{from=880, to=887},
|
|
{from=890, to=895},
|
|
{from=900, to=906},
|
|
{from=908, to=908},
|
|
{from=910, to=929},
|
|
{from=931, to=1154},
|
|
{from=1162, to=1279},
|
|
{from=7682, to=7683},
|
|
{from=7690, to=7691},
|
|
{from=7710, to=7711},
|
|
{from=7744, to=7745},
|
|
{from=7766, to=7767},
|
|
{from=7776, to=7777},
|
|
{from=7786, to=7787},
|
|
{from=7808, to=7813},
|
|
{from=7835, to=7835},
|
|
{from=7922, to=7923},
|
|
{from=8208, to=8208},
|
|
{from=8210, to=8231},
|
|
{from=8240, to=8286},
|
|
{from=8304, to=8305},
|
|
{from=8308, to=8334},
|
|
{from=8336, to=8348},
|
|
{from=8352, to=8383},
|
|
{from=8448, to=8587},
|
|
{from=8592, to=9254},
|
|
{from=9312, to=10239},
|
|
{from=10496, to=11007},
|
|
{from=64256, to=64262},
|
|
}
|
|
|
|
local function shouldCodepointBeEscaped(cp)
|
|
for _, range in ipairs(UNICODE_RANGES_NOT_TO_ESCAPE) do -- @Speed: Don't use a loop?
|
|
if cp >= range.from and cp <= range.to then return false end
|
|
end
|
|
return true
|
|
end
|
|
|
|
-- local cache = setmetatable({}, {__mode="kv"}) -- :SerializationCache (This doesn't seem to speed things up.)
|
|
|
|
-- success, error = serialize( buffer, value )
|
|
local function serialize(buffer, v)
|
|
--[[ :SerializationCache
|
|
if cache[v] then
|
|
tableInsert(buffer, cache[v])
|
|
return true
|
|
end
|
|
local bufferStart = #buffer + 1
|
|
--]]
|
|
|
|
local vType = type(v)
|
|
|
|
if vType == "table" then
|
|
local first = true
|
|
tableInsert(buffer, "{")
|
|
|
|
local indices = {}
|
|
for i, item in ipairs(v) do
|
|
if not first then tableInsert(buffer, ",") end
|
|
first = false
|
|
|
|
local ok, err = serialize(buffer, item)
|
|
if not ok then return false, err end
|
|
|
|
indices[i] = true
|
|
end
|
|
|
|
local keys = {}
|
|
for k, item in pairs(v) do
|
|
if indices[k] then
|
|
-- void
|
|
elseif type(k) == "table" then
|
|
return false, "Table keys cannot be tables."
|
|
else
|
|
tableInsert(keys, k)
|
|
end
|
|
end
|
|
|
|
table.sort(keys, function(a, b)
|
|
return tostring(a) < tostring(b)
|
|
end)
|
|
|
|
for _, k in ipairs(keys) do
|
|
local item = v[k]
|
|
|
|
if not first then tableInsert(buffer, ",") end
|
|
first = false
|
|
|
|
if not KEYWORDS[k] and type(k) == "string" and k:find"^[%a_][%w_]*$" then
|
|
tableInsert(buffer, k)
|
|
tableInsert(buffer, "=")
|
|
|
|
else
|
|
tableInsert(buffer, "[")
|
|
|
|
local ok, err = serialize(buffer, k)
|
|
if not ok then return false, err end
|
|
|
|
tableInsert(buffer, "]=")
|
|
end
|
|
|
|
local ok, err = serialize(buffer, item)
|
|
if not ok then return false, err end
|
|
end
|
|
|
|
tableInsert(buffer, "}")
|
|
|
|
elseif vType == "string" then
|
|
if v == "" then
|
|
tableInsert(buffer, '""')
|
|
return true
|
|
end
|
|
|
|
local useApostrophe = v:find('"', 1, true) and not v:find("'", 1, true)
|
|
local quote = useApostrophe and "'" or '"'
|
|
|
|
tableInsert(buffer, quote)
|
|
|
|
if current_anytime_fastStrings or not v:find"[^\32-\126\t\n]" then
|
|
-- print(">> FAST", #v) -- DEBUG
|
|
|
|
local s = v:gsub((useApostrophe and "[\t\n\\']" or '[\t\n\\"]'), function(c)
|
|
return ESCAPE_SEQUENCES[c] or internalError(c:byte())
|
|
end)
|
|
tableInsert(buffer, s)
|
|
|
|
else
|
|
-- print(">> SLOW", #v) -- DEBUG
|
|
local pos = 1
|
|
|
|
-- @Speed: There are optimizations to be made here!
|
|
while pos <= #v do
|
|
local c = v:sub(pos, pos)
|
|
local cp, len = utf8GetCodepointAndLength(v, pos)
|
|
|
|
-- Named escape sequences.
|
|
if ESCAPE_SEQUENCES_EXCEPT_QUOTES[c] then tableInsert(buffer, ESCAPE_SEQUENCES_EXCEPT_QUOTES[c]) ; pos = pos+1
|
|
elseif c == quote then tableInsert(buffer, [[\]]) ; tableInsert(buffer, quote) ; pos = pos+1
|
|
|
|
-- UTF-8 character.
|
|
elseif len == 1 and not shouldCodepointBeEscaped(cp) then tableInsert(buffer, v:sub(pos, pos )) ; pos = pos+1 -- @Speed: We can insert multiple single-byte characters sometimes!
|
|
elseif len and not shouldCodepointBeEscaped(cp) then tableInsert(buffer, v:sub(pos, pos+len-1)) ; pos = pos+len
|
|
|
|
-- Anything else.
|
|
else
|
|
tableInsert(buffer, F((v:find("^%d", pos+1) and "\\%03d" or "\\%d"), v:byte(pos)))
|
|
pos = pos + 1
|
|
end
|
|
end
|
|
end
|
|
|
|
tableInsert(buffer, quote)
|
|
|
|
elseif v == 1/0 then
|
|
tableInsert(buffer, "(1/0)")
|
|
elseif v == -1/0 then
|
|
tableInsert(buffer, "(-1/0)")
|
|
elseif v ~= v then
|
|
tableInsert(buffer, "(0/0)") -- NaN.
|
|
elseif v == 0 then
|
|
tableInsert(buffer, "0") -- In case it's actually -0 for some reason, which would be silly to output.
|
|
elseif vType == "number" then
|
|
if v < 0 then
|
|
tableInsert(buffer, " ") -- The space prevents an accidental comment if a "-" is right before.
|
|
end
|
|
tableInsert(buffer, tostring(v)) -- (I'm not sure what precision tostring() uses for numbers. Maybe we should use string.format() instead.)
|
|
|
|
elseif vType == "boolean" or v == nil then
|
|
tableInsert(buffer, tostring(v))
|
|
|
|
else
|
|
return false, F("Cannot serialize value of type '%s'. (%s)", vType, tostring(v))
|
|
end
|
|
|
|
--[[ :SerializationCache
|
|
if v ~= nil then
|
|
cache[v] = table.concat(buffer, "", bufferStart, #buffer)
|
|
end
|
|
--]]
|
|
|
|
return true
|
|
end
|
|
|
|
-- luaString = toLua( value )
|
|
-- Returns nil and a message on error.
|
|
local function toLua(v)
|
|
local buffer = {}
|
|
|
|
local ok, err = serialize(buffer, v)
|
|
if not ok then return nil, err end
|
|
|
|
return table.concat(buffer)
|
|
end
|
|
|
|
-- value = evaluate( expression [, environment=getfenv() ] )
|
|
-- Returns nil and a message on error.
|
|
local function evaluate(expr, env)
|
|
local chunk, err = loadLuaString("return("..expr.."\n)", "@<evaluate>", (env or getfenv(2)))
|
|
if not chunk then
|
|
return nil, F("Invalid expression '%s'. (%s)", expr, (err:gsub("^:%d+: ", "")))
|
|
end
|
|
|
|
local ok, valueOrErr = pcall(chunk)
|
|
if not ok then return nil, valueOrErr end
|
|
|
|
return valueOrErr -- May be nil or false!
|
|
end
|
|
|
|
|
|
|
|
local function escapePattern(s)
|
|
return (s:gsub("[-+*^?$.%%()[%]]", "%%%0"))
|
|
end
|
|
|
|
|
|
|
|
local function outputLineNumber(parts, ln)
|
|
tableInsert(parts, "--[[@")
|
|
tableInsert(parts, ln)
|
|
tableInsert(parts, "]]")
|
|
end
|
|
|
|
--[[local]] function maybeOutputLineNumber(parts, tok, lastLn)
|
|
if tok.line == lastLn or USELESS_TOKENS[tok.type] then return lastLn end
|
|
|
|
outputLineNumber(parts, tok.line)
|
|
return tok.line
|
|
end
|
|
--[=[
|
|
--[[local]] function maybeOutputLineNumber(parts, tok, lastLn, fromMetaToOutput)
|
|
if tok.line == lastLn or USELESS_TOKENS[tok.type] then return lastLn end
|
|
|
|
if fromMetaToOutput then
|
|
tableInsert(parts, '__LUA"--[[@'..tok.line..']]"\n')
|
|
else
|
|
tableInsert(parts, "--[[@"..tok.line.."]]")
|
|
end
|
|
return tok.line
|
|
end
|
|
]=]
|
|
|
|
|
|
|
|
local function isAny(v, ...)
|
|
for i = 1, select("#", ...) do
|
|
if v == select(i, ...) then return true end
|
|
end
|
|
return false
|
|
end
|
|
|
|
|
|
|
|
local function errorIfNotRunningMeta(level)
|
|
if not current_anytime_isRunningMeta then
|
|
error("No file is being processed.", 1+level)
|
|
end
|
|
end
|
|
|
|
|
|
|
|
local function copyArray(t)
|
|
local copy = {}
|
|
for i, v in ipairs(t) do
|
|
copy[i] = v
|
|
end
|
|
return copy
|
|
end
|
|
|
|
local copyTable
|
|
do
|
|
local function deepCopy(t, copy, tableCopies)
|
|
for k, v in pairs(t) do
|
|
if type(v) == "table" then
|
|
local vCopy = tableCopies[v]
|
|
|
|
if vCopy then
|
|
copy[k] = vCopy
|
|
else
|
|
vCopy = {}
|
|
tableCopies[v] = vCopy
|
|
copy[k] = deepCopy(v, vCopy, tableCopies)
|
|
end
|
|
|
|
else
|
|
copy[k] = v
|
|
end
|
|
end
|
|
return copy
|
|
end
|
|
|
|
-- copy = copyTable( table [, deep=false ] )
|
|
--[[local]] function copyTable(t, deep)
|
|
local copy = {}
|
|
|
|
if deep then
|
|
return deepCopy(t, copy, {[t]=copy})
|
|
end
|
|
|
|
for k, v in pairs(t) do copy[k] = v end
|
|
|
|
return copy
|
|
end
|
|
end
|
|
|
|
|
|
|
|
-- values = pack( value1, ... )
|
|
-- values.n is the amount of values (which can be zero).
|
|
local pack = (
|
|
(_VERSION >= "Lua 5.2" or jit) and table.pack
|
|
or function(...)
|
|
return {n=select("#", ...), ...}
|
|
end
|
|
)
|
|
|
|
local unpack = (_VERSION >= "Lua 5.2") and table.unpack or _G.unpack
|
|
|
|
|
|
|
|
--[[local]] loadLuaString = (
|
|
(_VERSION >= "Lua 5.2" or jit) and function(lua, chunkName, env)
|
|
return load(lua, chunkName, "bt", env)
|
|
end
|
|
or function(lua, chunkName, env)
|
|
local chunk, err = loadstring(lua, chunkName)
|
|
if not chunk then return nil, err end
|
|
|
|
if env then setfenv(chunk, env) end
|
|
|
|
return chunk
|
|
end
|
|
)
|
|
|
|
local loadLuaFile = (
|
|
(_VERSION >= "Lua 5.2" or jit) and function(path, env)
|
|
return loadfile(path, "bt", env)
|
|
end
|
|
or function(path, env)
|
|
local chunk, err = loadfile(path)
|
|
if not chunk then return nil, err end
|
|
|
|
if env then setfenv(chunk, env) end
|
|
|
|
return chunk
|
|
end
|
|
)
|
|
|
|
local function isLuaStringValidExpression(lua)
|
|
return loadLuaString("return("..lua.."\n)", "@", nil) ~= nil
|
|
end
|
|
|
|
|
|
|
|
-- token, index = getNextUsableToken( tokens, startIndex, indexLimit=autoDependingOnDirection, direction )
|
|
local function getNextUsableToken(tokens, iStart, iLimit, dir)
|
|
iLimit = (
|
|
dir < 0
|
|
and math.max((iLimit or 1 ), 1)
|
|
or math.min((iLimit or 1/0), #tokens)
|
|
)
|
|
|
|
for i = iStart, iLimit, dir do
|
|
if not USELESS_TOKENS[tokens[i].type] then
|
|
return tokens[i], i
|
|
end
|
|
end
|
|
|
|
return nil
|
|
end
|
|
|
|
|
|
|
|
-- bool = isToken( token, tokenType [, tokenValue=any ] )
|
|
local function isToken(tok, tokType, v)
|
|
return tok.type == tokType and (v == nil or tok.value == v)
|
|
end
|
|
|
|
-- bool = isTokenAndNotNil( token, tokenType [, tokenValue=any ] )
|
|
local function isTokenAndNotNil(tok, tokType, v)
|
|
return tok ~= nil and tok.type == tokType and (v == nil or tok.value == v)
|
|
end
|
|
|
|
|
|
|
|
--[[local]] function getLineNumber(s, pos)
|
|
return 1 + countSubString(s, 1, pos-1, "\n", true)
|
|
end
|
|
|
|
|
|
|
|
-- text = getRelativeLocationText( tokenOfInterest, otherToken )
|
|
-- text = getRelativeLocationText( tokenOfInterest, otherFilename, otherLineNumber )
|
|
local function getRelativeLocationText(tokOfInterest, otherFilename, otherLn)
|
|
if type(otherFilename) == "table" then
|
|
return getRelativeLocationText(tokOfInterest, otherFilename.file, otherFilename.line)
|
|
end
|
|
|
|
if not (tokOfInterest.file and tokOfInterest.line) then
|
|
return "at <UnknownLocation>"
|
|
end
|
|
|
|
if tokOfInterest.file ~= otherFilename then return F("at %s:%d", tokOfInterest.file, tokOfInterest.line) end
|
|
if tokOfInterest.line+1 == otherLn then return F("on the previous line") end
|
|
if tokOfInterest.line-1 == otherLn then return F("on the next line") end
|
|
if tokOfInterest.line ~= otherLn then return F("on line %d", tokOfInterest.line) end
|
|
return "on the same line"
|
|
end
|
|
|
|
|
|
|
|
--[[local]] tableInsert = table.insert
|
|
--[[local]] tableRemove = table.remove
|
|
|
|
--[[local]] function tableInsertFormat(t, s, ...)
|
|
tableInsert(t, F(s, ...))
|
|
end
|
|
|
|
|
|
|
|
-- length|nil = utf8GetCharLength( string [, position=1 ] )
|
|
local function utf8GetCharLength(s, pos)
|
|
pos = pos or 1
|
|
local b1, b2, b3, b4 = s:byte(pos, pos+3)
|
|
|
|
if b1 > 0 and b1 <= 127 then
|
|
return 1
|
|
|
|
elseif b1 >= 194 and b1 <= 223 then
|
|
if not b2 then return nil end -- UTF-8 string terminated early.
|
|
if b2 < 128 or b2 > 191 then return nil end -- Invalid UTF-8 character.
|
|
return 2
|
|
|
|
elseif b1 >= 224 and b1 <= 239 then
|
|
if not b3 then return nil end -- UTF-8 string terminated early.
|
|
if b1 == 224 and (b2 < 160 or b2 > 191) then return nil end -- Invalid UTF-8 character.
|
|
if b1 == 237 and (b2 < 128 or b2 > 159) then return nil end -- Invalid UTF-8 character.
|
|
if (b2 < 128 or b2 > 191) then return nil end -- Invalid UTF-8 character.
|
|
if (b3 < 128 or b3 > 191) then return nil end -- Invalid UTF-8 character.
|
|
return 3
|
|
|
|
elseif b1 >= 240 and b1 <= 244 then
|
|
if not b4 then return nil end -- UTF-8 string terminated early.
|
|
if b1 == 240 and (b2 < 144 or b2 > 191) then return nil end -- Invalid UTF-8 character.
|
|
if b1 == 244 and (b2 < 128 or b2 > 143) then return nil end -- Invalid UTF-8 character.
|
|
if (b2 < 128 or b2 > 191) then return nil end -- Invalid UTF-8 character.
|
|
if (b3 < 128 or b3 > 191) then return nil end -- Invalid UTF-8 character.
|
|
if (b4 < 128 or b4 > 191) then return nil end -- Invalid UTF-8 character.
|
|
return 4
|
|
end
|
|
|
|
return nil -- Invalid UTF-8 character.
|
|
end
|
|
|
|
-- codepoint, length = utf8GetCodepointAndLength( string [, position=1 ] )
|
|
-- Returns nil if the text is invalid at the position.
|
|
--[[local]] function utf8GetCodepointAndLength(s, pos)
|
|
pos = pos or 1
|
|
local len = utf8GetCharLength(s, pos)
|
|
if not len then return nil end
|
|
|
|
-- 2^6=64, 2^12=4096, 2^18=262144
|
|
if len == 1 then return s:byte(pos), len end
|
|
if len == 2 then local b1, b2 = s:byte(pos, pos+1) ; return (b1-192)*64 + (b2-128), len end
|
|
if len == 3 then local b1, b2, b3 = s:byte(pos, pos+2) ; return (b1-224)*4096 + (b2-128)*64 + (b3-128), len end
|
|
do local b1, b2, b3, b4 = s:byte(pos, pos+3) ; return (b1-240)*262144 + (b2-128)*4096 + (b3-128)*64 + (b4-128), len end
|
|
end
|
|
|
|
|
|
|
|
-- for k, v in pairsSorted( table ) do
|
|
local function pairsSorted(t)
|
|
local keys = {}
|
|
for k in pairs(t) do
|
|
tableInsert(keys, k)
|
|
end
|
|
sortNatural(keys)
|
|
|
|
local i = 0
|
|
|
|
return function()
|
|
i = i+1
|
|
local k = keys[i]
|
|
if k ~= nil then return k, t[k] end
|
|
end
|
|
end
|
|
|
|
|
|
|
|
-- sortNatural( array )
|
|
-- aIsLessThanB = compareNatural( a, b )
|
|
local compareNatural
|
|
do
|
|
local function pad(numStr)
|
|
return F("%03d%s", #numStr, numStr)
|
|
end
|
|
--[[local]] function compareNatural(a, b)
|
|
if type(a) == "number" and type(b) == "number" then
|
|
return a < b
|
|
else
|
|
return (tostring(a):gsub("%d+", pad) < tostring(b):gsub("%d+", pad))
|
|
end
|
|
end
|
|
|
|
--[[local]] function sortNatural(t, k)
|
|
table.sort(t, compareNatural)
|
|
end
|
|
end
|
|
|
|
|
|
|
|
-- lua = _loadResource( resourceName, isParsing==true , nameToken, stats ) -- At parse time.
|
|
-- lua = _loadResource( resourceName, isParsing==false, errorLevel ) -- At metaprogram runtime.
|
|
local function _loadResource(resourceName, isParsing, nameTokOrErrLevel, stats)
|
|
local lua = current_parsingAndMeta_resourceCache[resourceName]
|
|
|
|
if not lua then
|
|
if current_parsingAndMeta_onInsert then
|
|
lua = current_parsingAndMeta_onInsert(resourceName)
|
|
|
|
if type(lua) == "string" then
|
|
-- void
|
|
elseif isParsing then
|
|
errorAtToken(nameTokOrErrLevel, nameTokOrErrLevel.position+1, "Parser/MetaProgram", "Expected a string from params.onInsert(). (Got %s)", type(lua))
|
|
else
|
|
errorf(1+nameTokOrErrLevel, "Expected a string from params.onInsert(). (Got %s)", type(lua))
|
|
end
|
|
|
|
else
|
|
local err
|
|
lua, err = readFile(resourceName, true)
|
|
|
|
if lua then
|
|
-- void
|
|
elseif isParsing then
|
|
errorAtToken(nameTokOrErrLevel, nameTokOrErrLevel.position+1, "Parser", "Could not read file '%s'. (%s)", resourceName, tostring(err))
|
|
else
|
|
errorf(1+nameTokOrErrLevel, "Could not read file '%s'. (%s)", resourceName, tostring(err))
|
|
end
|
|
end
|
|
|
|
current_parsingAndMeta_resourceCache[resourceName] = lua
|
|
|
|
if isParsing then
|
|
tableInsert(stats.insertedNames, resourceName)
|
|
end
|
|
|
|
elseif isParsing then
|
|
current_parsing_insertCount = current_parsing_insertCount + 1 -- Note: We don't count insertions of newly encountered files.
|
|
|
|
if current_parsing_insertCount > MAX_DUPLICATE_FILE_INSERTS then
|
|
errorAtToken(
|
|
nameTokOrErrLevel, nameTokOrErrLevel.position+1, "Parser",
|
|
"Too many duplicate inserts. We may be stuck in a recursive loop. (Unique files inserted so far: %s)",
|
|
stats.insertedNames[1] and table.concat(stats.insertedNames, ", ") or "none"
|
|
)
|
|
end
|
|
end
|
|
|
|
return lua
|
|
end
|
|
|
|
|
|
|
|
--==============================================================
|
|
--= Preprocessor Functions =====================================
|
|
--==============================================================
|
|
|
|
|
|
|
|
-- :EnvironmentTable
|
|
----------------------------------------------------------------
|
|
|
|
metaEnv = copyTable(_G) -- Include all standard Lua stuff.
|
|
metaEnv._G = metaEnv
|
|
|
|
local metaFuncs = {}
|
|
|
|
-- printf()
|
|
-- printf( format, value1, ... )
|
|
-- Print a formatted string to stdout.
|
|
metaFuncs.printf = printf
|
|
|
|
-- readFile()
|
|
-- contents = readFile( path [, isTextFile=false ] )
|
|
-- Get the entire contents of a binary file or text file. Returns nil and a message on error.
|
|
metaFuncs.readFile = readFile
|
|
metaFuncs.getFileContents = readFile -- @Deprecated
|
|
|
|
-- writeFile()
|
|
-- success, error = writeFile( path, contents ) -- Writes a binary file.
|
|
-- success, error = writeFile( path, isTextFile, contents )
|
|
-- Write an entire binary file or text file.
|
|
metaFuncs.writeFile = writeFile
|
|
|
|
-- fileExists()
|
|
-- bool = fileExists( path )
|
|
-- Check if a file exists.
|
|
metaFuncs.fileExists = fileExists
|
|
|
|
-- toLua()
|
|
-- luaString = toLua( value )
|
|
-- Convert a value to a Lua literal. Does not work with certain types, like functions or userdata.
|
|
-- Returns nil and a message on error.
|
|
metaFuncs.toLua = toLua
|
|
|
|
-- serialize()
|
|
-- success, error = serialize( buffer, value )
|
|
-- Same as toLua() except adds the result to an array instead of returning the Lua code as a string.
|
|
-- This could avoid allocating unnecessary strings.
|
|
metaFuncs.serialize = serialize
|
|
|
|
-- evaluate()
|
|
-- value = evaluate( expression [, environment=getfenv() ] )
|
|
-- Evaluate a Lua expression. The function is kind of the opposite of toLua(). Returns nil and a message on error.
|
|
-- Note that nil or false can also be returned as the first value if that's the value the expression results in!
|
|
metaFuncs.evaluate = evaluate
|
|
|
|
-- escapePattern()
|
|
-- escapedString = escapePattern( string )
|
|
-- Escape a string so it can be used in a pattern as plain text.
|
|
metaFuncs.escapePattern = escapePattern
|
|
|
|
-- isToken()
|
|
-- bool = isToken( token, tokenType [, tokenValue=any ] )
|
|
-- Check if a token is of a specific type, optionally also check it's value.
|
|
metaFuncs.isToken = isToken
|
|
|
|
-- copyTable()
|
|
-- copy = copyTable( table [, deep=false ] )
|
|
-- Copy a table, optionally recursively (deep copy).
|
|
-- Multiple references to the same table and self-references are preserved during deep copying.
|
|
metaFuncs.copyTable = copyTable
|
|
|
|
-- unpack()
|
|
-- value1, ... = unpack( array [, fromIndex=1, toIndex=#array ] )
|
|
-- Is _G.unpack() in Lua 5.1 and alias for table.unpack() in Lua 5.2+.
|
|
metaFuncs.unpack = unpack
|
|
|
|
-- pack()
|
|
-- values = pack( value1, ... )
|
|
-- Put values in a new array. values.n is the amount of values (which can be zero)
|
|
-- including nil values. Alias for table.pack() in Lua 5.2+.
|
|
metaFuncs.pack = pack
|
|
|
|
-- pairsSorted()
|
|
-- for key, value in pairsSorted( table ) do
|
|
-- Same as pairs() but the keys are sorted (ascending).
|
|
metaFuncs.pairsSorted = pairsSorted
|
|
|
|
-- sortNatural()
|
|
-- sortNatural( array )
|
|
-- Sort an array using compareNatural().
|
|
metaFuncs.sortNatural = sortNatural
|
|
|
|
-- compareNatural()
|
|
-- aIsLessThanB = compareNatural( a, b )
|
|
-- Compare two strings. Numbers in the strings are compared as numbers (as opposed to as strings).
|
|
-- Examples:
|
|
-- print( "foo9" < "foo10" ) -- false
|
|
-- print(compareNatural("foo9", "foo10")) -- true
|
|
metaFuncs.compareNatural = compareNatural
|
|
|
|
-- run()
|
|
-- returnValue1, ... = run( path [, arg1, ... ] )
|
|
-- Execute a Lua file. Similar to dofile().
|
|
function metaFuncs.run(path, ...)
|
|
assertarg(1, path, "string")
|
|
|
|
local main_chunk, err = loadLuaFile(path, metaEnv)
|
|
if not main_chunk then error(err, 0) end
|
|
|
|
-- We want multiple return values while avoiding a tail call to preserve stack info.
|
|
local returnValues = pack(main_chunk(...))
|
|
return unpack(returnValues, 1, returnValues.n)
|
|
end
|
|
|
|
-- outputValue()
|
|
-- outputValue( value )
|
|
-- outputValue( value1, value2, ... ) -- Outputted values will be separated by commas.
|
|
-- Output one or more values, like strings or tables, as literals.
|
|
-- Raises an error if no file or string is being processed.
|
|
function metaFuncs.outputValue(...)
|
|
errorIfNotRunningMeta(2)
|
|
|
|
local argCount = select("#", ...)
|
|
if argCount == 0 then
|
|
error("No values to output.", 2)
|
|
end
|
|
|
|
for i = 1, argCount do
|
|
local v = select(i, ...)
|
|
|
|
if v == nil and not current_meta_canOutputNil then
|
|
local ln = debug.getinfo(2, "l").currentline
|
|
errorOnLine(current_meta_pathForErrorMessages, ln, "MetaProgram", "Trying to output nil which is disallowed through params.canOutputNil .")
|
|
end
|
|
|
|
if i > 1 then
|
|
tableInsert(current_meta_output, (current_parsingAndMeta_isDebug and ", " or ","))
|
|
end
|
|
|
|
local ok, err = serialize(current_meta_output, v)
|
|
|
|
if not ok then
|
|
local ln = debug.getinfo(2, "l").currentline
|
|
errorOnLine(current_meta_pathForErrorMessages, ln, "MetaProgram", "%s", err)
|
|
end
|
|
end
|
|
end
|
|
|
|
-- outputLua()
|
|
-- outputLua( luaString1, ... )
|
|
-- Output one or more strings as raw Lua code.
|
|
-- Raises an error if no file or string is being processed.
|
|
function metaFuncs.outputLua(...)
|
|
errorIfNotRunningMeta(2)
|
|
|
|
local argCount = select("#", ...)
|
|
if argCount == 0 then
|
|
error("No Lua code to output.", 2)
|
|
end
|
|
|
|
for i = 1, argCount do
|
|
local lua = select(i, ...)
|
|
assertarg(i, lua, "string")
|
|
tableInsert(current_meta_output, lua)
|
|
end
|
|
end
|
|
|
|
-- outputLuaTemplate()
|
|
-- outputLuaTemplate( luaStringTemplate, value1, ... )
|
|
-- Use a string as a template for outputting Lua code with values.
|
|
-- Question marks (?) are replaced with the values.
|
|
-- Raises an error if no file or string is being processed.
|
|
-- Examples:
|
|
-- outputLuaTemplate("local name, age = ?, ?", "Harry", 48)
|
|
-- outputLuaTemplate("dogs[?] = ?", "greyhound", {italian=false, count=5})
|
|
function metaFuncs.outputLuaTemplate(lua, ...)
|
|
errorIfNotRunningMeta(2)
|
|
assertarg(1, lua, "string")
|
|
|
|
local args = {...} -- @Memory
|
|
local n = 0
|
|
local v, err
|
|
|
|
lua = lua:gsub("%?", function()
|
|
n = n + 1
|
|
v, err = toLua(args[n])
|
|
|
|
if not v then
|
|
errorf(3, "Bad argument %d: %s", 1+n, err)
|
|
end
|
|
|
|
return v
|
|
end)
|
|
|
|
tableInsert(current_meta_output, lua)
|
|
end
|
|
|
|
-- getOutputSoFar()
|
|
-- luaString = getOutputSoFar( [ asTable=false ] )
|
|
-- getOutputSoFar( buffer )
|
|
-- Get Lua code that's been outputted so far.
|
|
-- If asTable is false then the full Lua code string is returned.
|
|
-- If asTable is true then an array of Lua code segments is returned. (This avoids allocating, possibly large, strings.)
|
|
-- If a buffer array is given then Lua code segments are added to it.
|
|
-- Raises an error if no file or string is being processed.
|
|
function metaFuncs.getOutputSoFar(bufferOrAsTable)
|
|
errorIfNotRunningMeta(2)
|
|
|
|
-- Should there be a way to get the contents of current_meta_output etc.? :GetMoreOutputFromStack
|
|
|
|
if type(bufferOrAsTable) == "table" then
|
|
for _, lua in ipairs(current_meta_outputStack[1]) do
|
|
tableInsert(bufferOrAsTable, lua)
|
|
end
|
|
-- Return nothing!
|
|
|
|
else
|
|
return bufferOrAsTable and copyArray(current_meta_outputStack[1]) or table.concat(current_meta_outputStack[1])
|
|
end
|
|
end
|
|
|
|
local lineFragments = {}
|
|
|
|
local function getOutputSoFarOnLine()
|
|
errorIfNotRunningMeta(2)
|
|
|
|
local len = 0
|
|
|
|
-- Should there be a way to get the contents of current_meta_output etc.? :GetMoreOutputFromStack
|
|
for i = #current_meta_outputStack[1], 1, -1 do
|
|
local fragment = current_meta_outputStack[1][i]
|
|
|
|
if fragment:find("\n", 1, true) then
|
|
len = len + 1
|
|
lineFragments[len] = fragment:gsub(".*\n", "")
|
|
break
|
|
end
|
|
|
|
len = len + 1
|
|
lineFragments[len] = fragment
|
|
end
|
|
|
|
return table.concat(lineFragments, 1, len)
|
|
end
|
|
|
|
-- getOutputSoFarOnLine()
|
|
-- luaString = getOutputSoFarOnLine( )
|
|
-- Get Lua code that's been outputted so far on the current line.
|
|
-- Raises an error if no file or string is being processed.
|
|
metaFuncs.getOutputSoFarOnLine = getOutputSoFarOnLine
|
|
|
|
-- getOutputSizeSoFar()
|
|
-- size = getOutputSizeSoFar( )
|
|
-- Get the amount of bytes outputted so far.
|
|
-- Raises an error if no file or string is being processed.
|
|
function metaFuncs.getOutputSizeSoFar()
|
|
errorIfNotRunningMeta(2)
|
|
|
|
local size = 0
|
|
|
|
for _, lua in ipairs(current_meta_outputStack[1]) do -- :GetMoreOutputFromStack
|
|
size = size + #lua
|
|
end
|
|
|
|
return size
|
|
end
|
|
|
|
-- getCurrentLineNumberInOutput()
|
|
-- lineNumber = getCurrentLineNumberInOutput( )
|
|
-- Get the current line number in the output.
|
|
function metaFuncs.getCurrentLineNumberInOutput()
|
|
errorIfNotRunningMeta(2)
|
|
|
|
local ln = 1
|
|
|
|
for _, lua in ipairs(current_meta_outputStack[1]) do -- :GetMoreOutputFromStack
|
|
ln = ln + countString(lua, "\n", true)
|
|
end
|
|
|
|
return ln
|
|
end
|
|
|
|
local function getIndentation(line, tabWidth)
|
|
if not tabWidth then
|
|
return line:match"^[ \t]*"
|
|
end
|
|
|
|
local indent = 0
|
|
|
|
for i = 1, #line do
|
|
if line:sub(i, i) == "\t" then
|
|
indent = math.floor(indent/tabWidth)*tabWidth + tabWidth
|
|
elseif line:sub(i, i) == " " then
|
|
indent = indent + 1
|
|
else
|
|
break
|
|
end
|
|
end
|
|
|
|
return indent
|
|
end
|
|
|
|
-- getIndentation()
|
|
-- string = getIndentation( line )
|
|
-- size = getIndentation( line, tabWidth )
|
|
-- Get indentation of a line, either as a string or as a size in spaces.
|
|
metaFuncs.getIndentation = getIndentation
|
|
|
|
-- getCurrentIndentationInOutput()
|
|
-- string = getCurrentIndentationInOutput( )
|
|
-- size = getCurrentIndentationInOutput( tabWidth )
|
|
-- Get the indentation of the current line, either as a string or as a size in spaces.
|
|
function metaFuncs.getCurrentIndentationInOutput(tabWidth)
|
|
errorIfNotRunningMeta(2)
|
|
return (getIndentation(getOutputSoFarOnLine(), tabWidth))
|
|
end
|
|
|
|
-- getCurrentPathIn()
|
|
-- path = getCurrentPathIn( )
|
|
-- Get what file is currently being processed, if any.
|
|
function metaFuncs.getCurrentPathIn()
|
|
return current_anytime_pathIn
|
|
end
|
|
|
|
-- getCurrentPathOut()
|
|
-- path = getCurrentPathOut( )
|
|
-- Get what file the currently processed file will be written to, if any.
|
|
function metaFuncs.getCurrentPathOut()
|
|
return current_anytime_pathOut
|
|
end
|
|
|
|
-- tokenize()
|
|
-- tokens = tokenize( luaString [, allowPreprocessorCode=false ] )
|
|
-- token = {
|
|
-- type=tokenType, representation=representation, value=value,
|
|
-- line=lineNumber, lineEnd=lineNumber, position=bytePosition, file=filePath,
|
|
-- ...
|
|
-- }
|
|
-- Convert Lua code to tokens. Returns nil and a message on error. (See newToken() for token types.)
|
|
function metaFuncs.tokenize(lua, allowPpCode)
|
|
local ok, errOrTokens = pcall(_tokenize, lua, "<string>", allowPpCode, allowPpCode, true) -- @Incomplete: Make allowJitSyntax a parameter to tokenize()?
|
|
if not ok then
|
|
return nil, cleanError(errOrTokens)
|
|
end
|
|
return errOrTokens
|
|
end
|
|
|
|
-- removeUselessTokens()
|
|
-- removeUselessTokens( tokens )
|
|
-- Remove whitespace and comment tokens.
|
|
function metaFuncs.removeUselessTokens(tokens)
|
|
local len = #tokens
|
|
local offset = 0
|
|
|
|
for i, tok in ipairs(tokens) do
|
|
if USELESS_TOKENS[tok.type] then
|
|
offset = offset-1
|
|
else
|
|
tokens[i+offset] = tok
|
|
end
|
|
end
|
|
|
|
for i = len, len+offset+1, -1 do
|
|
tokens[i] = nil
|
|
end
|
|
end
|
|
|
|
local function nextUsefulToken(tokens, i)
|
|
while true do
|
|
i = i+1
|
|
local tok = tokens[i]
|
|
if not tok then return end
|
|
if not USELESS_TOKENS[tok.type] then return i, tok end
|
|
end
|
|
end
|
|
|
|
-- eachToken()
|
|
-- for index, token in eachToken( tokens [, ignoreUselessTokens=false ] ) do
|
|
-- Loop through tokens.
|
|
function metaFuncs.eachToken(tokens, ignoreUselessTokens)
|
|
if ignoreUselessTokens then
|
|
return nextUsefulToken, tokens, 0
|
|
else
|
|
return ipairs(tokens)
|
|
end
|
|
end
|
|
|
|
-- getNextUsefulToken()
|
|
-- token, index = getNextUsefulToken( tokens, startIndex [, steps=1 ] )
|
|
-- Get the next token that isn't a whitespace or comment. Returns nil if no more tokens are found.
|
|
-- Specify a negative steps value to get an earlier token.
|
|
function metaFuncs.getNextUsefulToken(tokens, i1, steps)
|
|
steps = (steps or 1)
|
|
|
|
local i2, dir
|
|
if steps == 0 then
|
|
return tokens[i1], i1
|
|
elseif steps < 0 then
|
|
i2, dir = 1, -1
|
|
else
|
|
i2, dir = #tokens, 1
|
|
end
|
|
|
|
for i = i1, i2, dir do
|
|
local tok = tokens[i]
|
|
if not USELESS_TOKENS[tok.type] then
|
|
steps = steps-dir
|
|
if steps == 0 then return tok, i end
|
|
end
|
|
end
|
|
|
|
return nil
|
|
end
|
|
|
|
local numberFormatters = {
|
|
auto = function(n) return tostring(n) end,
|
|
integer = function(n) return F("%d", n) end,
|
|
int = function(n) return F("%d", n) end,
|
|
float = function(n) return F("%f", n):gsub("(%d)0+$", "%1") end,
|
|
scientific = function(n) return F("%e", n):gsub("(%d)0+e", "%1e"):gsub("0+(%d+)$", "%1") end,
|
|
SCIENTIFIC = function(n) return F("%E", n):gsub("(%d)0+E", "%1E"):gsub("0+(%d+)$", "%1") end,
|
|
e = function(n) return F("%e", n):gsub("(%d)0+e", "%1e"):gsub("0+(%d+)$", "%1") end,
|
|
E = function(n) return F("%E", n):gsub("(%d)0+E", "%1E"):gsub("0+(%d+)$", "%1") end,
|
|
hexadecimal = function(n) return (n == math.floor(n) and F("0x%x", n) or error("Hexadecimal floats not supported yet.", 3)) end, -- @Incomplete
|
|
HEXADECIMAL = function(n) return (n == math.floor(n) and F("0x%X", n) or error("Hexadecimal floats not supported yet.", 3)) end,
|
|
hex = function(n) return (n == math.floor(n) and F("0x%x", n) or error("Hexadecimal floats not supported yet.", 3)) end,
|
|
HEX = function(n) return (n == math.floor(n) and F("0x%X", n) or error("Hexadecimal floats not supported yet.", 3)) end,
|
|
}
|
|
|
|
-- newToken()
|
|
-- token = newToken( tokenType, ... )
|
|
-- Create a new token. Different token types take different arguments.
|
|
--
|
|
-- commentToken = newToken( "comment", contents [, forceLongForm=false ] )
|
|
-- identifierToken = newToken( "identifier", identifier )
|
|
-- keywordToken = newToken( "keyword", keyword )
|
|
-- numberToken = newToken( "number", number [, numberFormat="auto" ] )
|
|
-- punctuationToken = newToken( "punctuation", symbol )
|
|
-- stringToken = newToken( "string", contents [, longForm=false ] )
|
|
-- whitespaceToken = newToken( "whitespace", contents )
|
|
-- ppEntryToken = newToken( "pp_entry", isDouble )
|
|
-- ppKeywordToken = newToken( "pp_keyword", ppKeyword ) -- ppKeyword can be "file", "insert", "line" or "@".
|
|
-- ppSymbolToken = newToken( "pp_symbol", identifier )
|
|
--
|
|
-- commentToken = { type="comment", representation=string, value=string, long=isLongForm }
|
|
-- identifierToken = { type="identifier", representation=string, value=string }
|
|
-- keywordToken = { type="keyword", representation=string, value=string }
|
|
-- numberToken = { type="number", representation=string, value=number }
|
|
-- punctuationToken = { type="punctuation", representation=string, value=string }
|
|
-- stringToken = { type="string", representation=string, value=string, long=isLongForm }
|
|
-- whitespaceToken = { type="whitespace", representation=string, value=string }
|
|
-- ppEntryToken = { type="pp_entry", representation=string, value=string, double=isDouble }
|
|
-- ppKeywordToken = { type="pp_keyword", representation=string, value=string }
|
|
-- ppSymbolToken = { type="pp_symbol", representation=string, value=string }
|
|
--
|
|
-- Number formats:
|
|
-- "integer" E.g. 42
|
|
-- "int" Same as integer, e.g. 42
|
|
-- "float" E.g. 3.14
|
|
-- "scientific" E.g. 0.7e+12
|
|
-- "SCIENTIFIC" E.g. 0.7E+12 (upper case)
|
|
-- "e" Same as scientific, e.g. 0.7e+12
|
|
-- "E" Same as SCIENTIFIC, e.g. 0.7E+12 (upper case)
|
|
-- "hexadecimal" E.g. 0x19af
|
|
-- "HEXADECIMAL" E.g. 0x19AF (upper case)
|
|
-- "hex" Same as hexadecimal, e.g. 0x19af
|
|
-- "HEX" Same as HEXADECIMAL, e.g. 0x19AF (upper case)
|
|
-- "auto" Note: Infinite numbers and NaN always get automatic format.
|
|
--
|
|
function metaFuncs.newToken(tokType, ...)
|
|
if tokType == "comment" then
|
|
local comment, long = ...
|
|
long = not not (long or comment:find"[\r\n]")
|
|
assertarg(2, comment, "string")
|
|
|
|
local repr
|
|
if long then
|
|
local equalSigns = ""
|
|
|
|
while comment:find(F("]%s]", equalSigns), 1, true) do
|
|
equalSigns = equalSigns.."="
|
|
end
|
|
|
|
repr = F("--[%s[%s]%s]", equalSigns, comment, equalSigns)
|
|
|
|
else
|
|
repr = F("--%s\n", comment)
|
|
end
|
|
|
|
return {type="comment", representation=repr, value=comment, long=long}
|
|
|
|
elseif tokType == "identifier" then
|
|
local ident = ...
|
|
assertarg(2, ident, "string")
|
|
|
|
if ident == "" then
|
|
error("Identifier length is 0.", 2)
|
|
elseif not ident:find"^[%a_][%w_]*$" then
|
|
errorf(2, "Bad identifier format: '%s'", ident)
|
|
elseif KEYWORDS[ident] then
|
|
errorf(2, "Identifier must not be a keyword: '%s'", ident)
|
|
end
|
|
|
|
return {type="identifier", representation=ident, value=ident}
|
|
|
|
elseif tokType == "keyword" then
|
|
local keyword = ...
|
|
assertarg(2, keyword, "string")
|
|
|
|
if not KEYWORDS[keyword] then
|
|
errorf(2, "Bad keyword '%s'.", keyword)
|
|
end
|
|
|
|
return {type="keyword", representation=keyword, value=keyword}
|
|
|
|
elseif tokType == "number" then
|
|
local n, numberFormat = ...
|
|
numberFormat = numberFormat or "auto"
|
|
assertarg(2, n, "number")
|
|
assertarg(3, numberFormat, "string")
|
|
|
|
-- Some of these are technically multiple other tokens. We could raise an error but ehhh...
|
|
local numStr = (
|
|
n ~= n and "(0/0)" or
|
|
n == 1/0 and "(1/0)" or
|
|
n == -1/0 and "(-1/0)" or
|
|
numberFormatters[numberFormat] and numberFormatters[numberFormat](n) or
|
|
errorf(2, "Invalid number format '%s'.", numberFormat)
|
|
)
|
|
|
|
return {type="number", representation=numStr, value=n}
|
|
|
|
elseif tokType == "punctuation" then
|
|
local symbol = ...
|
|
assertarg(2, symbol, "string")
|
|
|
|
-- Note: "!" and "!!" are of a different token type (pp_entry).
|
|
if not PUNCTUATION[symbol] then
|
|
errorf(2, "Bad symbol '%s'.", symbol)
|
|
end
|
|
|
|
return {type="punctuation", representation=symbol, value=symbol}
|
|
|
|
elseif tokType == "string" then
|
|
local s, long = ...
|
|
long = not not long
|
|
assertarg(2, s, "string")
|
|
|
|
local repr
|
|
|
|
if long then
|
|
local equalSigns = ""
|
|
|
|
while s:find(F("]%s]", equalSigns), 1, true) do
|
|
equalSigns = equalSigns .. "="
|
|
end
|
|
|
|
repr = F("[%s[%s]%s]", equalSigns, s, equalSigns)
|
|
|
|
else
|
|
repr = toLua(s)
|
|
end
|
|
|
|
return {type="string", representation=repr, value=s, long=long}
|
|
|
|
elseif tokType == "whitespace" then
|
|
local whitespace = ...
|
|
assertarg(2, whitespace, "string")
|
|
|
|
if whitespace == "" then
|
|
error("String is empty.", 2)
|
|
elseif whitespace:find"%S" then
|
|
error("String contains non-whitespace characters.", 2)
|
|
end
|
|
|
|
return {type="whitespace", representation=whitespace, value=whitespace}
|
|
|
|
elseif tokType == "pp_entry" then
|
|
local double = ...
|
|
assertarg(2, double, "boolean")
|
|
|
|
local symbol = double and "!!" or "!"
|
|
|
|
return {type="pp_entry", representation=symbol, value=symbol, double=double}
|
|
|
|
elseif tokType == "pp_keyword" then
|
|
local keyword = ...
|
|
assertarg(2, keyword, "string")
|
|
|
|
if keyword == "@" then
|
|
return {type="pp_keyword", representation="@@", value="insert"}
|
|
elseif not PREPROCESSOR_KEYWORDS[keyword] then
|
|
errorf(2, "Bad preprocessor keyword '%s'.", keyword)
|
|
else
|
|
return {type="pp_keyword", representation="@"..keyword, value=keyword}
|
|
end
|
|
|
|
elseif tokType == "pp_symbol" then
|
|
local ident = ...
|
|
assertarg(2, ident, "string")
|
|
|
|
if ident == "" then
|
|
error("Identifier length is 0.", 2)
|
|
elseif not ident:find"^[%a_][%w_]*$" then
|
|
errorf(2, "Bad identifier format: '%s'", ident)
|
|
elseif KEYWORDS[ident] then
|
|
errorf(2, "Identifier must not be a keyword: '%s'", ident)
|
|
else
|
|
return {type="pp_symbol", representation="$"..ident, value=ident}
|
|
end
|
|
|
|
else
|
|
errorf(2, "Invalid token type '%s'.", tostring(tokType))
|
|
end
|
|
end
|
|
|
|
-- concatTokens()
|
|
-- luaString = concatTokens( tokens )
|
|
-- Concatenate tokens by their representations.
|
|
function metaFuncs.concatTokens(tokens)
|
|
return (_concatTokens(tokens, nil, false, nil, nil))
|
|
end
|
|
|
|
local recycledArrays = {}
|
|
|
|
-- startInterceptingOutput()
|
|
-- startInterceptingOutput( )
|
|
-- Start intercepting output until stopInterceptingOutput() is called.
|
|
-- The function can be called multiple times to intercept interceptions.
|
|
function metaFuncs.startInterceptingOutput()
|
|
errorIfNotRunningMeta(2)
|
|
|
|
current_meta_output = tableRemove(recycledArrays) or {}
|
|
for i = 1, #current_meta_output do current_meta_output[i] = nil end
|
|
tableInsert(current_meta_outputStack, current_meta_output)
|
|
end
|
|
|
|
local function _stopInterceptingOutput(errLevel)
|
|
errorIfNotRunningMeta(1+errLevel)
|
|
|
|
local interceptedLua = tableRemove(current_meta_outputStack)
|
|
current_meta_output = current_meta_outputStack[#current_meta_outputStack] or error("Called stopInterceptingOutput() before calling startInterceptingOutput().", 1+errLevel)
|
|
tableInsert(recycledArrays, interceptedLua)
|
|
|
|
return table.concat(interceptedLua)
|
|
end
|
|
|
|
-- stopInterceptingOutput()
|
|
-- luaString = stopInterceptingOutput( )
|
|
-- Stop intercepting output and retrieve collected code.
|
|
function metaFuncs.stopInterceptingOutput()
|
|
return (_stopInterceptingOutput(2))
|
|
end
|
|
|
|
-- loadResource()
|
|
-- luaString = loadResource( name )
|
|
-- Load a Lua file/resource (using the same mechanism as @insert"name").
|
|
-- Note that resources are cached after loading once.
|
|
function metaFuncs.loadResource(resourceName)
|
|
errorIfNotRunningMeta(2)
|
|
|
|
return (_loadResource(resourceName, false, 2))
|
|
end
|
|
|
|
local function isCallable(v)
|
|
return type(v) == "function"
|
|
-- We use debug.getmetatable instead of _G.getmetatable because we don't want to
|
|
-- potentially invoke user code - we just want to know if the value is callable.
|
|
or (type(v) == "table" and debug.getmetatable(v) ~= nil and type(debug.getmetatable(v).__call) == "function")
|
|
end
|
|
|
|
-- callMacro()
|
|
-- luaString = callMacro( function|macroName, argument1, ... )
|
|
-- Call a macro function (which must be a global in metaEnvironment if macroName is given).
|
|
-- The arguments should be Lua code strings.
|
|
function metaFuncs.callMacro(nameOrFunc, ...)
|
|
errorIfNotRunningMeta(2)
|
|
|
|
assertarg(1, nameOrFunc, "string","function")
|
|
local f
|
|
|
|
if type(nameOrFunc) == "string" then
|
|
local nameResult = current_parsingAndMeta_macroPrefix .. nameOrFunc .. current_parsingAndMeta_macroSuffix
|
|
f = metaEnv[nameResult]
|
|
|
|
if not isCallable(f) then
|
|
if nameOrFunc == nameResult
|
|
then errorf(2, "'%s' is not a macro/global function. (Got %s)", nameOrFunc, type(f))
|
|
else errorf(2, "'%s' (resolving to '%s') is not a macro/global function. (Got %s)", nameOrFunc, nameResult, type(f)) end
|
|
end
|
|
|
|
else
|
|
f = nameOrFunc
|
|
end
|
|
|
|
return (metaEnv.__M()(f(...)))
|
|
end
|
|
|
|
-- isProcessing()
|
|
-- bool = isProcessing( )
|
|
-- Returns true if a file or string is currently being processed.
|
|
function metaFuncs.isProcessing()
|
|
return current_parsingAndMeta_isProcessing
|
|
end
|
|
|
|
-- :PredefinedMacros
|
|
|
|
-- ASSERT()
|
|
-- @@ASSERT( condition [, message=auto ] )
|
|
-- Macro. Does nothing if params.release is set, otherwise calls error() if the
|
|
-- condition fails. The message argument is only evaluated if the condition fails.
|
|
function metaFuncs.ASSERT(conditionCode, messageCode)
|
|
errorIfNotRunningMeta(2)
|
|
if not conditionCode then error("missing argument #1 to 'ASSERT'", 2) end
|
|
|
|
-- if not isLuaStringValidExpression(conditionCode) then
|
|
-- errorf(2, "Invalid condition expression: %s", formatCodeForShortMessage(conditionCode))
|
|
-- end
|
|
|
|
if current_meta_releaseMode then return end
|
|
|
|
tableInsert(current_meta_output, "if not (")
|
|
tableInsert(current_meta_output, conditionCode)
|
|
tableInsert(current_meta_output, ") then error(")
|
|
|
|
if messageCode then
|
|
tableInsert(current_meta_output, "(")
|
|
tableInsert(current_meta_output, messageCode)
|
|
tableInsert(current_meta_output, ")")
|
|
else
|
|
tableInsert(current_meta_output, F("%q", "Assertion failed: "..conditionCode))
|
|
end
|
|
|
|
tableInsert(current_meta_output, ") end")
|
|
end
|
|
|
|
-- LOG()
|
|
-- @@LOG( logLevel, value ) -- [1]
|
|
-- @@LOG( logLevel, format, value1, ... ) -- [2]
|
|
--
|
|
-- Macro. Does nothing if logLevel is lower than params.logLevel,
|
|
-- otherwise prints a value[1] or a formatted message[2].
|
|
--
|
|
-- logLevel can be "error", "warning", "info", "debug" or "trace"
|
|
-- (from highest to lowest priority).
|
|
--
|
|
function metaFuncs.LOG(logLevelCode, valueOrFormatCode, ...)
|
|
errorIfNotRunningMeta(2)
|
|
if not logLevelCode then error("missing argument #1 to 'LOG'", 2) end
|
|
if not valueOrFormatCode then error("missing argument #2 to 'LOG'", 2) end
|
|
|
|
local chunk = loadLuaString("return("..logLevelCode.."\n)", "@", dummyEnv)
|
|
if not chunk then errorf(2, "Invalid logLevel expression: %s", formatCodeForShortMessage(logLevelCode)) end
|
|
|
|
local ok, logLevel = pcall(chunk)
|
|
if not ok then errorf(2, "logLevel must be a constant expression. Got: %s", formatCodeForShortMessage(logLevelCode)) end
|
|
if not LOG_LEVELS[logLevel] then errorf(2, "Invalid logLevel '%s'.", tostring(logLevel)) end
|
|
if logLevel == "off" then errorf(2, "Invalid logLevel '%s'.", tostring(logLevel)) end
|
|
|
|
if LOG_LEVELS[logLevel] > LOG_LEVELS[current_meta_maxLogLevel] then return end
|
|
|
|
tableInsert(current_meta_output, "print(")
|
|
|
|
if ... then
|
|
tableInsert(current_meta_output, "string.format(")
|
|
tableInsert(current_meta_output, valueOrFormatCode)
|
|
for i = 1, select("#", ...) do
|
|
tableInsert(current_meta_output, ", ")
|
|
tableInsert(current_meta_output, (select(i, ...)))
|
|
end
|
|
tableInsert(current_meta_output, ")")
|
|
else
|
|
tableInsert(current_meta_output, valueOrFormatCode)
|
|
end
|
|
|
|
tableInsert(current_meta_output, ")")
|
|
end
|
|
|
|
-- Extra stuff used by the command line program:
|
|
metaFuncs.tryToFormatError = tryToFormatError
|
|
|
|
----------------------------------------------------------------
|
|
|
|
|
|
|
|
for k, v in pairs(metaFuncs) do metaEnv[k] = v end
|
|
|
|
metaEnv.__LUA = metaEnv.outputLua
|
|
metaEnv.__VAL = metaEnv.outputValue
|
|
|
|
function metaEnv.__TOLUA(v)
|
|
return (assert(toLua(v)))
|
|
end
|
|
function metaEnv.__ISLUA(lua)
|
|
if type(lua) ~= "string" then
|
|
error("Value is not Lua code.", 2)
|
|
end
|
|
return lua
|
|
end
|
|
|
|
local function finalizeMacro(lua)
|
|
if lua == nil then
|
|
return (_stopInterceptingOutput(2))
|
|
elseif type(lua) ~= "string" then
|
|
errorf(2, "[Macro] Value is not Lua code. (Got %s)", type(lua))
|
|
elseif current_meta_output[1] then
|
|
error("[Macro] Got Lua code from both value expression and outputLua(). Only one method may be used.", 2) -- It's also possible interception calls are unbalanced.
|
|
else
|
|
_stopInterceptingOutput(2) -- Returns "" because nothing was outputted.
|
|
return lua
|
|
end
|
|
end
|
|
function metaEnv.__M()
|
|
metaFuncs.startInterceptingOutput()
|
|
return finalizeMacro
|
|
end
|
|
|
|
-- luaString = __ARG( locationTokenNumber, luaString|callback )
|
|
-- callback = function( )
|
|
function metaEnv.__ARG(locTokNum, v)
|
|
local lua
|
|
if type(v) == "string" then
|
|
lua = v
|
|
else
|
|
metaFuncs.startInterceptingOutput()
|
|
v()
|
|
lua = _stopInterceptingOutput(2)
|
|
end
|
|
|
|
if current_parsingAndMeta_strictMacroArguments and not isLuaStringValidExpression(lua) then
|
|
runtimeErrorAtToken(2, current_meta_locationTokens[locTokNum], nil, "MacroArgument", "Argument result is not a valid Lua expression: %s", formatCodeForShortMessage(lua))
|
|
end
|
|
|
|
return lua
|
|
end
|
|
|
|
function metaEnv.__EVAL(v) -- For symbols.
|
|
if isCallable(v) then
|
|
v = v()
|
|
end
|
|
return v
|
|
end
|
|
|
|
|
|
|
|
local function getLineCountWithCode(tokens)
|
|
local lineCount = 0
|
|
local lastLine = 0
|
|
|
|
for _, tok in ipairs(tokens) do
|
|
if not USELESS_TOKENS[tok.type] and tok.lineEnd > lastLine then
|
|
lineCount = lineCount+(tok.lineEnd-tok.line+1)
|
|
lastLine = tok.lineEnd
|
|
end
|
|
end
|
|
|
|
return lineCount
|
|
end
|
|
|
|
|
|
|
|
--
|
|
-- Preprocessor expansions (symbols etc., not macros).
|
|
--
|
|
|
|
local function newTokenAt(tok, locTok)
|
|
tok.line = tok.line or locTok and locTok.line
|
|
tok.lineEnd = tok.lineEnd or locTok and locTok.lineEnd
|
|
tok.position = tok.position or locTok and locTok.position
|
|
tok.file = tok.file or locTok and locTok.file
|
|
return tok
|
|
end
|
|
|
|
local function popTokens(tokenStack, lastIndexToPop)
|
|
for i = #tokenStack, lastIndexToPop, -1 do
|
|
tokenStack[i] = nil
|
|
end
|
|
end
|
|
local function popUseless(tokenStack)
|
|
for i = #tokenStack, 1, -1 do
|
|
if not USELESS_TOKENS[tokenStack[i].type] then break end
|
|
tokenStack[i] = nil
|
|
end
|
|
end
|
|
|
|
local function advanceToken(tokens)
|
|
local tok = tokens[tokens.nextI]
|
|
tokens.nextI = tokens.nextI + 1
|
|
return tok
|
|
end
|
|
local function advancePastUseless(tokens)
|
|
for i = tokens.nextI, #tokens do
|
|
if not USELESS_TOKENS[tokens[i].type] then break end
|
|
tokens.nextI = i + 1
|
|
end
|
|
end
|
|
|
|
-- outTokens = doEarlyExpansions( tokensToExpand, stats )
|
|
local function doEarlyExpansions(tokensToExpand, stats)
|
|
--
|
|
-- Here we expand simple things that makes it easier for
|
|
-- doLateExpansions*() to do more elaborate expansions.
|
|
--
|
|
-- Expand expressions:
|
|
-- @file
|
|
-- @line
|
|
-- ` ... `
|
|
-- $symbol
|
|
--
|
|
local tokenStack = {} -- We process the last token first, and we may push new tokens onto the stack.
|
|
local outTokens = {}
|
|
|
|
for i = #tokensToExpand, 1, -1 do
|
|
tableInsert(tokenStack, tokensToExpand[i])
|
|
end
|
|
|
|
while tokenStack[1] do
|
|
local tok = tokenStack[#tokenStack]
|
|
|
|
-- Keyword.
|
|
if isToken(tok, "pp_keyword") then
|
|
local ppKeywordTok = tok
|
|
|
|
-- @file
|
|
-- @line
|
|
if ppKeywordTok.value == "file" then
|
|
tableRemove(tokenStack) -- '@file'
|
|
tableInsert(outTokens, newTokenAt({type="string", value=ppKeywordTok.file, representation=F("%q",ppKeywordTok.file)}, ppKeywordTok))
|
|
elseif ppKeywordTok.value == "line" then
|
|
tableRemove(tokenStack) -- '@line'
|
|
tableInsert(outTokens, newTokenAt({type="number", value=ppKeywordTok.line, representation=F(" %d ",ppKeywordTok.line)}, ppKeywordTok)) -- Is it fine for the representation to have spaces? Probably.
|
|
|
|
else
|
|
-- Expand later.
|
|
tableInsert(outTokens, ppKeywordTok)
|
|
tableRemove(tokenStack) -- '@...'
|
|
end
|
|
|
|
-- Backtick string.
|
|
elseif isToken(tok, "string") and tok.representation:find"^`" then
|
|
local stringTok = tok
|
|
stringTok.representation = toLua(stringTok.value)--F("%q", stringTok.value)
|
|
|
|
tableInsert(outTokens, stringTok)
|
|
tableRemove(tokenStack) -- the string
|
|
|
|
-- Symbol. (Should this expand later? Does it matter? Yeah, do this in the AST code instead. @Cleanup)
|
|
elseif isToken(tok, "pp_symbol") then
|
|
local ppSymbolTok = tok
|
|
|
|
-- $symbol
|
|
tableRemove(tokenStack) -- '$symbol'
|
|
tableInsert(outTokens, newTokenAt({type="pp_entry", value="!!", representation="!!", double=true}, ppSymbolTok))
|
|
tableInsert(outTokens, newTokenAt({type="punctuation", value="(", representation="(" }, ppSymbolTok))
|
|
tableInsert(outTokens, newTokenAt({type="identifier", value="__EVAL", representation="__EVAL" }, ppSymbolTok))
|
|
tableInsert(outTokens, newTokenAt({type="punctuation", value="(", representation="(" }, ppSymbolTok))
|
|
tableInsert(outTokens, newTokenAt({type="identifier", value=ppSymbolTok.value, representation=ppSymbolTok.value}, ppSymbolTok))
|
|
tableInsert(outTokens, newTokenAt({type="punctuation", value=")", representation=")" }, ppSymbolTok))
|
|
tableInsert(outTokens, newTokenAt({type="punctuation", value=")", representation=")" }, ppSymbolTok))
|
|
|
|
-- Anything else.
|
|
else
|
|
tableInsert(outTokens, tok)
|
|
tableRemove(tokenStack) -- anything
|
|
end
|
|
end--while tokenStack
|
|
|
|
return outTokens
|
|
end
|
|
|
|
-- outTokens = doLateExpansions( tokensToExpand, stats, allowBacktickStrings, allowJitSyntax )
|
|
local function doLateExpansions(tokensToExpand, stats, allowBacktickStrings, allowJitSyntax)
|
|
--
|
|
-- Expand expressions:
|
|
-- @insert "name"
|
|
--
|
|
local tokenStack = {} -- We process the last token first, and we may push new tokens onto the stack.
|
|
local outTokens = {}
|
|
|
|
for i = #tokensToExpand, 1, -1 do
|
|
tableInsert(tokenStack, tokensToExpand[i])
|
|
end
|
|
|
|
while tokenStack[1] do
|
|
local tok = tokenStack[#tokenStack]
|
|
|
|
-- Keyword.
|
|
if isToken(tok, "pp_keyword") then
|
|
local ppKeywordTok = tok
|
|
local tokNext, iNext = getNextUsableToken(tokenStack, #tokenStack-1, nil, -1)
|
|
|
|
-- @insert "name"
|
|
if ppKeywordTok.value == "insert" and isTokenAndNotNil(tokNext, "string") and tokNext.file == ppKeywordTok.file then
|
|
local nameTok = tokNext
|
|
popTokens(tokenStack, iNext) -- the string
|
|
|
|
local toInsertName = nameTok.value
|
|
local toInsertLua = _loadResource(toInsertName, true, nameTok, stats)
|
|
local toInsertTokens = _tokenize(toInsertLua, toInsertName, true, allowBacktickStrings, allowJitSyntax)
|
|
toInsertTokens = doEarlyExpansions(toInsertTokens, stats)
|
|
|
|
for i = #toInsertTokens, 1, -1 do
|
|
tableInsert(tokenStack, toInsertTokens[i])
|
|
end
|
|
|
|
local lastTok = toInsertTokens[#toInsertTokens]
|
|
stats.processedByteCount = stats.processedByteCount + #toInsertLua
|
|
stats.lineCount = stats.lineCount + (lastTok and lastTok.line + countString(lastTok.representation, "\n", true) or 0)
|
|
stats.lineCountCode = stats.lineCountCode + getLineCountWithCode(toInsertTokens)
|
|
|
|
-- @insert identifier ( argument1, ... )
|
|
-- @insert identifier " ... "
|
|
-- @insert identifier { ... }
|
|
-- @insert identifier !( ... )
|
|
-- @insert identifier !!( ... )
|
|
elseif ppKeywordTok.value == "insert" and isTokenAndNotNil(tokNext, "identifier") and tokNext.file == ppKeywordTok.file then
|
|
local identTok = tokNext
|
|
tokNext, iNext = getNextUsableToken(tokenStack, iNext-1, nil, -1)
|
|
|
|
if not (tokNext and (
|
|
tokNext.type == "string"
|
|
or (tokNext.type == "punctuation" and isAny(tokNext.value, "(","{",".",":","["))
|
|
or tokNext.type == "pp_entry"
|
|
)) then
|
|
errorAtToken(identTok, identTok.position+#identTok.representation, "Parser/Macro", "Expected '(' after macro name '%s'.", identTok.value)
|
|
end
|
|
|
|
-- Expand later.
|
|
tableInsert(outTokens, tok)
|
|
tableRemove(tokenStack) -- '@insert'
|
|
|
|
elseif ppKeywordTok.value == "insert" then
|
|
errorAtToken(
|
|
ppKeywordTok, (tokNext and tokNext.position or ppKeywordTok.position+#ppKeywordTok.representation),
|
|
"Parser", "Expected a string or identifier after %s.", ppKeywordTok.representation
|
|
)
|
|
|
|
else
|
|
errorAtToken(ppKeywordTok, nil, "Parser", "Internal error. (%s)", ppKeywordTok.value)
|
|
end
|
|
|
|
-- Anything else.
|
|
else
|
|
tableInsert(outTokens, tok)
|
|
tableRemove(tokenStack) -- anything
|
|
end
|
|
end--while tokenStack
|
|
|
|
return outTokens
|
|
end
|
|
|
|
-- outTokens = doExpansions( params, tokensToExpand, stats )
|
|
local function doExpansions(params, tokens, stats)
|
|
tokens = doEarlyExpansions(tokens, stats)
|
|
tokens = doLateExpansions (tokens, stats, params.backtickStrings, params.jitSyntax) -- Resources.
|
|
return tokens
|
|
end
|
|
|
|
|
|
|
|
--
|
|
-- Metaprogram generation.
|
|
--
|
|
|
|
local function AstSequence(locTok, tokens) return {
|
|
type = "sequence",
|
|
locationToken = locTok,
|
|
nodes = tokens or {},
|
|
} end
|
|
local function AstLua(locTok, tokens) return { -- plain Lua
|
|
type = "lua",
|
|
locationToken = locTok,
|
|
tokens = tokens or {},
|
|
} end
|
|
local function AstMetaprogram(locTok, tokens) return { -- `!(statements)` or `!statements`
|
|
type = "metaprogram",
|
|
locationToken = locTok,
|
|
originIsLine = false,
|
|
tokens = tokens or {},
|
|
} end
|
|
local function AstExpressionCode(locTok, tokens) return { -- `!!(expression)`
|
|
type = "expressionCode",
|
|
locationToken = locTok,
|
|
tokens = tokens or {},
|
|
} end
|
|
local function AstExpressionValue(locTok, tokens) return { -- `!(expression)`
|
|
type = "expressionValue",
|
|
locationToken = locTok,
|
|
tokens = tokens or {},
|
|
} end
|
|
local function AstDualCode(locTok, valueTokens) return { -- `!!declaration` or `!!assignment`
|
|
type = "dualCode",
|
|
locationToken = locTok,
|
|
isDeclaration = false,
|
|
names = {},
|
|
valueTokens = valueTokens or {},
|
|
} end
|
|
-- local function AstSymbol(locTok) return { -- `$name`
|
|
-- type = "symbol",
|
|
-- locationToken = locTok,
|
|
-- name = "",
|
|
-- } end
|
|
local function AstMacro(locTok, calleeTokens) return { -- `@@callee(arguments)` or `@@callee{}` or `@@callee""`
|
|
type = "macro",
|
|
locationToken = locTok,
|
|
calleeTokens = calleeTokens or {},
|
|
arguments = {}, -- []MacroArgument
|
|
} end
|
|
local function MacroArgument(locTok, nodes) return {
|
|
locationToken = locTok,
|
|
isComplex = false,
|
|
nodes = nodes or {},
|
|
} end
|
|
|
|
local astParseMetaBlockOrLine
|
|
|
|
local function astParseMetaBlock(tokens)
|
|
local ppEntryTokIndex = tokens.nextI
|
|
local ppEntryTok = tokens[ppEntryTokIndex]
|
|
tokens.nextI = tokens.nextI + 2 -- '!(' or '!!('
|
|
|
|
local outTokens = {}
|
|
local depthStack = {}
|
|
|
|
while true do
|
|
local tok = tokens[tokens.nextI]
|
|
|
|
if not tok then
|
|
if depthStack[1] then
|
|
tok = depthStack[#depthStack].startToken
|
|
errorAtToken(tok, nil, "Parser/MetaBlock", "Could not find matching bracket before EOF. (Preprocessor line starts %s)", getRelativeLocationText(ppEntryTok, tok))
|
|
end
|
|
break
|
|
end
|
|
|
|
-- End of meta block.
|
|
if not depthStack[1] and isToken(tok, "punctuation", ")") then
|
|
tokens.nextI = tokens.nextI + 1 -- after ')'
|
|
break
|
|
|
|
-- Nested metaprogram (not supported).
|
|
elseif tok.type:find"^pp_" then
|
|
errorAtToken(tok, nil, "Parser/MetaBlock", "Preprocessor token inside metaprogram (starting %s).", getRelativeLocationText(ppEntryTok, tok))
|
|
|
|
-- Continuation of meta block.
|
|
else
|
|
if isToken(tok, "punctuation", "(") then
|
|
tableInsert(depthStack, {startToken=tok, --[[1]]"punctuation", --[[2]]")"})
|
|
elseif isToken(tok, "punctuation", "[") then
|
|
tableInsert(depthStack, {startToken=tok, --[[1]]"punctuation", --[[2]]"]"})
|
|
elseif isToken(tok, "punctuation", "{") then
|
|
tableInsert(depthStack, {startToken=tok, --[[1]]"punctuation", --[[2]]"}"})
|
|
|
|
elseif
|
|
isToken(tok, "punctuation", ")") or
|
|
isToken(tok, "punctuation", "]") or
|
|
isToken(tok, "punctuation", "}")
|
|
then
|
|
if not depthStack[1] then
|
|
errorAtToken(tok, nil, "Parser/MetaBlock", "Unexpected '%s'. (Preprocessor line starts %s)", tok.value, getRelativeLocationText(ppEntryTok, tok))
|
|
elseif not isToken(tok, unpack(depthStack[#depthStack])) then
|
|
local startTok = depthStack[#depthStack].startToken
|
|
errorAtToken(
|
|
tok, nil, "Parser/MetaBlock", "Expected '%s' (to close '%s' %s) but got '%s'. (Preprocessor line starts %s)",
|
|
depthStack[#depthStack][2], startTok.value, getRelativeLocationText(startTok, tok), tok.value, getRelativeLocationText(ppEntryTok, tok)
|
|
)
|
|
end
|
|
tableRemove(depthStack)
|
|
end
|
|
|
|
tableInsert(outTokens, tok)
|
|
tokens.nextI = tokens.nextI + 1 -- after anything
|
|
end
|
|
end
|
|
|
|
local lua = _concatTokens(outTokens, nil, false, nil, nil)
|
|
local chunk, err = loadLuaString("return 0,"..lua.."\n,0", "@", nil)
|
|
local isExpression = (chunk ~= nil)
|
|
|
|
if not isExpression and ppEntryTok.double then
|
|
errorAtToken(tokens[ppEntryTokIndex+1], nil, "Parser/MetaBlock", "Invalid expression in preprocessor block.")
|
|
-- err = err:gsub("^:%d+: ", "")
|
|
-- errorAtToken(tokens[ppEntryTokIndex+1], nil, "Parser/MetaBlock", "Invalid expression in preprocessor block. (%s)", err)
|
|
elseif isExpression and not isLuaStringValidExpression(lua) then
|
|
if #lua > 100 then
|
|
lua = lua:sub(1, 50) .. "..." .. lua:sub(-50)
|
|
end
|
|
errorAtToken(tokens[ppEntryTokIndex+1], nil, "Parser/MetaBlock", "Ambiguous expression '%s'. (Comma-separated list?)", formatCodeForShortMessage(lua))
|
|
end
|
|
|
|
local astOutNode = ((ppEntryTok.double and AstExpressionCode) or (isExpression and AstExpressionValue or AstMetaprogram))(ppEntryTok, outTokens)
|
|
return astOutNode
|
|
end
|
|
|
|
local function astParseMetaLine(tokens)
|
|
local ppEntryTok = tokens[tokens.nextI]
|
|
tokens.nextI = tokens.nextI + 1 -- '!' or '!!'
|
|
|
|
local isDual = ppEntryTok.double
|
|
local astOutNode = (isDual and AstDualCode or AstMetaprogram)(ppEntryTok)
|
|
|
|
if astOutNode.type == "metaprogram" then
|
|
astOutNode.originIsLine = true
|
|
end
|
|
|
|
if isDual then
|
|
-- We expect the statement to look like any of these:
|
|
-- !!local x, y = ...
|
|
-- !!x, y = ...
|
|
local tokNext, iNext = getNextUsableToken(tokens, tokens.nextI, nil, 1)
|
|
|
|
if isTokenAndNotNil(tokNext, "keyword", "local") then
|
|
astOutNode.isDeclaration = true
|
|
|
|
tokens.nextI = iNext + 1 -- after 'local'
|
|
tokNext, iNext = getNextUsableToken(tokens, tokens.nextI, nil, 1)
|
|
end
|
|
|
|
local usedNames = {}
|
|
|
|
while true do
|
|
if not isTokenAndNotNil(tokNext, "identifier") then
|
|
local tok = tokNext or tokens[#tokens]
|
|
errorAtToken(
|
|
tok, nil, "Parser/DualCodeLine", "Expected %sidentifier. (Preprocessor line starts %s)",
|
|
(astOutNode.names[1] and "" or "'local' or "),
|
|
getRelativeLocationText(ppEntryTok, tok)
|
|
)
|
|
elseif usedNames[tokNext.value] then
|
|
errorAtToken(
|
|
tokNext, nil, "Parser/DualCodeLine", "Duplicate name '%s' in %s. (Preprocessor line starts %s)",
|
|
tokNext.value,
|
|
(astOutNode.isDeclaration and "declaration" or "assignment"),
|
|
getRelativeLocationText(ppEntryTok, tokNext)
|
|
)
|
|
end
|
|
tableInsert(astOutNode.names, tokNext.value)
|
|
usedNames[tokNext.value] = tokNext
|
|
tokens.nextI = iNext + 1 -- after the identifier
|
|
tokNext, iNext = getNextUsableToken(tokens, tokens.nextI, nil, 1)
|
|
|
|
if not isTokenAndNotNil(tokNext, "punctuation", ",") then break end
|
|
tokens.nextI = iNext + 1 -- after ','
|
|
tokNext, iNext = getNextUsableToken(tokens, tokens.nextI, nil, 1)
|
|
end
|
|
|
|
if not isTokenAndNotNil(tokNext, "punctuation", "=") then
|
|
local tok = tokNext or tokens[#tokens]
|
|
errorAtToken(
|
|
tok, nil, "Parser/DualCodeLine", "Expected '=' in %s. (Preprocessor line starts %s)",
|
|
(astOutNode.isDeclaration and "declaration" or "assignment"),
|
|
getRelativeLocationText(ppEntryTok, tok)
|
|
)
|
|
end
|
|
tokens.nextI = iNext + 1 -- after '='
|
|
end
|
|
|
|
-- Find end of metaprogram line.
|
|
local outTokens = isDual and astOutNode.valueTokens or astOutNode.tokens
|
|
local depthStack = {}
|
|
|
|
while true do
|
|
local tok = tokens[tokens.nextI]
|
|
|
|
if not tok then
|
|
if depthStack[1] then
|
|
tok = depthStack[#depthStack].startToken
|
|
errorAtToken(tok, nil, "Parser/MetaLine", "Could not find matching bracket before EOF. (Preprocessor line starts %s)", getRelativeLocationText(ppEntryTok, tok))
|
|
end
|
|
break
|
|
end
|
|
|
|
-- End of meta line.
|
|
if
|
|
not depthStack[1] and (
|
|
(tok.type == "whitespace" and tok.value:find("\n", 1, true)) or
|
|
(tok.type == "comment" and not tok.long)
|
|
)
|
|
then
|
|
tableInsert(outTokens, tok)
|
|
tokens.nextI = tokens.nextI + 1 -- after the whitespace or comment
|
|
break
|
|
|
|
-- Nested metaprogram (not supported).
|
|
elseif tok.type:find"^pp_" then
|
|
errorAtToken(tok, nil, "Parser/MetaLine", "Preprocessor token inside metaprogram (starting %s).", getRelativeLocationText(ppEntryTok, tok))
|
|
|
|
-- Continuation of meta line.
|
|
else
|
|
if isToken(tok, "punctuation", "(") then
|
|
tableInsert(depthStack, {startToken=tok, --[[1]]"punctuation", --[[2]]")"})
|
|
elseif isToken(tok, "punctuation", "[") then
|
|
tableInsert(depthStack, {startToken=tok, --[[1]]"punctuation", --[[2]]"]"})
|
|
elseif isToken(tok, "punctuation", "{") then
|
|
tableInsert(depthStack, {startToken=tok, --[[1]]"punctuation", --[[2]]"}"})
|
|
|
|
elseif
|
|
isToken(tok, "punctuation", ")") or
|
|
isToken(tok, "punctuation", "]") or
|
|
isToken(tok, "punctuation", "}")
|
|
then
|
|
if not depthStack[1] then
|
|
errorAtToken(tok, nil, "Parser/MetaLine", "Unexpected '%s'. (Preprocessor line starts %s)", tok.value, getRelativeLocationText(ppEntryTok, tok))
|
|
elseif not isToken(tok, unpack(depthStack[#depthStack])) then
|
|
local startTok = depthStack[#depthStack].startToken
|
|
errorAtToken(
|
|
tok, nil, "Parser/MetaLine", "Expected '%s' (to close '%s' %s) but got '%s'. (Preprocessor line starts %s)",
|
|
depthStack[#depthStack][2], startTok.value, getRelativeLocationText(startTok, tok), tok.value, getRelativeLocationText(ppEntryTok, tok)
|
|
)
|
|
end
|
|
tableRemove(depthStack)
|
|
end
|
|
|
|
tableInsert(outTokens, tok)
|
|
tokens.nextI = tokens.nextI + 1 -- after anything
|
|
end
|
|
end
|
|
|
|
return astOutNode
|
|
end
|
|
|
|
--[[local]] function astParseMetaBlockOrLine(tokens)
|
|
return isTokenAndNotNil(tokens[tokens.nextI+1], "punctuation", "(")
|
|
and astParseMetaBlock(tokens)
|
|
or astParseMetaLine (tokens)
|
|
end
|
|
|
|
local function astParseMacro(params, tokens)
|
|
local macroStartTok = tokens[tokens.nextI]
|
|
tokens.nextI = tokens.nextI + 1 -- after '@insert'
|
|
|
|
local astMacro = AstMacro(macroStartTok)
|
|
|
|
--
|
|
-- Callee.
|
|
--
|
|
|
|
-- Add 'ident' for start of (or whole) callee.
|
|
local tokNext, iNext = getNextUsableToken(tokens, tokens.nextI, nil, 1)
|
|
if not isTokenAndNotNil(tokNext, "identifier") then
|
|
printErrorTraceback("Internal error.")
|
|
errorAtToken(tokNext, nil, "Parser/Macro", "Internal error. (%s)", (tokNext and tokNext.type or "?"))
|
|
end
|
|
tokens.nextI = iNext + 1 -- after the identifier
|
|
tableInsert(astMacro.calleeTokens, tokNext)
|
|
local initialCalleeIdentTok = tokNext
|
|
|
|
-- Add macro prefix and suffix. (Note: We only edit the initial identifier in the callee if there are more.)
|
|
initialCalleeIdentTok.value = current_parsingAndMeta_macroPrefix .. initialCalleeIdentTok.value .. current_parsingAndMeta_macroSuffix
|
|
initialCalleeIdentTok.representation = initialCalleeIdentTok.value
|
|
|
|
-- Maybe add '.field[expr]:method' for rest of callee.
|
|
tokNext, iNext = getNextUsableToken(tokens, tokens.nextI, nil, 1)
|
|
|
|
while tokNext do
|
|
if isToken(tokNext, "punctuation", ".") or isToken(tokNext, "punctuation", ":") then
|
|
local punctTok = tokNext
|
|
tokens.nextI = iNext + 1 -- after '.' or ':'
|
|
tableInsert(astMacro.calleeTokens, tokNext)
|
|
|
|
tokNext, iNext = getNextUsableToken(tokens, tokens.nextI, nil, 1)
|
|
if not tokNext then
|
|
errorAfterToken(punctTok, "Parser/Macro", "Expected an identifier after '%s'.", punctTok.value)
|
|
end
|
|
tokens.nextI = iNext + 1 -- after the identifier
|
|
tableInsert(astMacro.calleeTokens, tokNext)
|
|
|
|
tokNext, iNext = getNextUsableToken(tokens, tokens.nextI, nil, 1)
|
|
|
|
if punctTok.value == ":" then break end
|
|
|
|
elseif isToken(tokNext, "punctuation", "[") then
|
|
local punctTok = tokNext
|
|
tokens.nextI = iNext + 1 -- after '['
|
|
tableInsert(astMacro.calleeTokens, tokNext)
|
|
|
|
local bracketBalance = 1
|
|
|
|
while true do
|
|
tokNext = advanceToken(tokens) -- anything
|
|
if not tokNext then
|
|
errorAtToken(punctTok, nil, "Parser/Macro", "Could not find matching bracket before EOF. (Macro starts %s)", getRelativeLocationText(macroStartTok, punctTok))
|
|
end
|
|
tableInsert(astMacro.calleeTokens, tokNext)
|
|
|
|
if isToken(tokNext, "punctuation", "[") then
|
|
bracketBalance = bracketBalance + 1
|
|
elseif isToken(tokNext, "punctuation", "]") then
|
|
bracketBalance = bracketBalance - 1
|
|
if bracketBalance == 0 then break end
|
|
elseif tokNext.type:find"^pp_" then
|
|
errorAtToken(tokNext, nil, "Parser/Macro", "Preprocessor token inside metaprogram/macro name expression (starting %s).", getRelativeLocationText(macroStartTok, tokNext))
|
|
end
|
|
end
|
|
|
|
tokNext, iNext = getNextUsableToken(tokens, tokens.nextI, nil, 1)
|
|
|
|
-- @UX: Validate that the contents form an expression.
|
|
|
|
else
|
|
break
|
|
end
|
|
end
|
|
|
|
--
|
|
-- Arguments.
|
|
--
|
|
|
|
-- @insert identifier " ... "
|
|
if isTokenAndNotNil(tokNext, "string") then
|
|
tableInsert(astMacro.arguments, MacroArgument(tokNext, {AstLua(tokNext, {tokNext})})) -- The one and only argument for this macro variant.
|
|
tokens.nextI = iNext + 1 -- after the string
|
|
|
|
-- @insert identifier { ... } -- Same as: @insert identifier ( { ... } )
|
|
elseif isTokenAndNotNil(tokNext, "punctuation", "{") then
|
|
local macroArg = MacroArgument(tokNext) -- The one and only argument for this macro variant.
|
|
astMacro.arguments[1] = macroArg
|
|
|
|
local astLuaInCurrentArg = AstLua(tokNext, {tokNext})
|
|
tableInsert(macroArg.nodes, astLuaInCurrentArg)
|
|
|
|
tokens.nextI = iNext + 1 -- after '{'
|
|
|
|
--
|
|
-- (Similar code as `@insert identifier()` below.)
|
|
--
|
|
|
|
-- Collect tokens for the table arg.
|
|
-- We're looking for the closing '}'.
|
|
local bracketDepth = 1 -- @Incomplete: Track all brackets!
|
|
|
|
while true do
|
|
local tok = tokens[tokens.nextI]
|
|
|
|
if not tok then
|
|
errorAtToken(macroArg.locationToken, nil, "Parser/MacroArgument", "Could not find end of table constructor before EOF.")
|
|
|
|
-- Preprocessor block in macro.
|
|
elseif tok.type == "pp_entry" then
|
|
tableInsert(macroArg.nodes, astParseMetaBlockOrLine(tokens))
|
|
astLuaInCurrentArg = nil
|
|
|
|
-- Nested macro.
|
|
elseif isToken(tok, "pp_keyword", "insert") then
|
|
tableInsert(macroArg.nodes, astParseMacro(params, tokens))
|
|
astLuaInCurrentArg = nil
|
|
|
|
-- Other preprocessor code in macro. (Not sure we ever get here.)
|
|
elseif tok.type:find"^pp_" then
|
|
errorAtToken(tok, nil, "Parser/MacroArgument", "Unsupported preprocessor code. (Macro starts %s)", getRelativeLocationText(macroStartTok, tok))
|
|
|
|
-- End of table and argument.
|
|
elseif bracketDepth == 1 and isToken(tok, "punctuation", "}") then
|
|
if not astLuaInCurrentArg then
|
|
astLuaInCurrentArg = AstLua(tok)
|
|
tableInsert(macroArg.nodes, astLuaInCurrentArg)
|
|
end
|
|
tableInsert(astLuaInCurrentArg.tokens, tok)
|
|
advanceToken(tokens) -- '}'
|
|
break
|
|
|
|
-- Normal token.
|
|
else
|
|
if isToken(tok, "punctuation", "{") then
|
|
bracketDepth = bracketDepth + 1
|
|
elseif isToken(tok, "punctuation", "}") then
|
|
bracketDepth = bracketDepth - 1
|
|
end
|
|
|
|
if not astLuaInCurrentArg then
|
|
astLuaInCurrentArg = AstLua(tok)
|
|
tableInsert(macroArg.nodes, astLuaInCurrentArg)
|
|
end
|
|
tableInsert(astLuaInCurrentArg.tokens, tok)
|
|
advanceToken(tokens) -- anything
|
|
end
|
|
end
|
|
|
|
-- @insert identifier ( argument1, ... )
|
|
elseif isTokenAndNotNil(tokNext, "punctuation", "(") then
|
|
-- Apply the same 'ambiguous syntax' rule as Lua. (Will comments mess this check up? @Check)
|
|
if isTokenAndNotNil(tokens[iNext-1], "whitespace") and tokens[iNext-1].value:find("\n", 1, true) then
|
|
errorAtToken(tokNext, nil, "Parser/Macro", "Ambiguous syntax near '(' - part of macro, or new statement?")
|
|
end
|
|
|
|
local parensStartTok = tokNext
|
|
tokens.nextI = iNext + 1 -- after '('
|
|
tokNext, iNext = getNextUsableToken(tokens, tokens.nextI, nil, 1)
|
|
|
|
if isTokenAndNotNil(tokNext, "punctuation", ")") then
|
|
tokens.nextI = iNext + 1 -- after ')'
|
|
|
|
else
|
|
for argNum = 1, 1/0 do
|
|
-- Collect tokens for this arg.
|
|
-- We're looking for the next comma at depth 0 or closing ')'.
|
|
local macroArg = MacroArgument(tokens[tokens.nextI])
|
|
astMacro.arguments[argNum] = macroArg
|
|
|
|
advancePastUseless(tokens) -- Trim leading useless tokens.
|
|
|
|
local astLuaInCurrentArg = nil
|
|
local depthStack = {}
|
|
|
|
while true do
|
|
local tok = tokens[tokens.nextI]
|
|
|
|
if not tok then
|
|
errorAtToken(parensStartTok, nil, "Parser/Macro", "Could not find end of argument list before EOF.")
|
|
|
|
-- Preprocessor block in macro.
|
|
elseif tok.type == "pp_entry" then
|
|
tableInsert(macroArg.nodes, astParseMetaBlockOrLine(tokens))
|
|
astLuaInCurrentArg = nil
|
|
|
|
-- Nested macro.
|
|
elseif isToken(tok, "pp_keyword", "insert") then
|
|
tableInsert(macroArg.nodes, astParseMacro(params, tokens))
|
|
astLuaInCurrentArg = nil
|
|
|
|
-- Other preprocessor code in macro. (Not sure we ever get here.)
|
|
elseif tok.type:find"^pp_" then
|
|
errorAtToken(tok, nil, "Parser/MacroArgument", "Unsupported preprocessor code. (Macro starts %s)", getRelativeLocationText(macroStartTok, tok))
|
|
|
|
-- End of argument.
|
|
elseif not depthStack[1] and (isToken(tok, "punctuation", ",") or isToken(tok, "punctuation", ")")) then
|
|
break
|
|
|
|
-- Normal token.
|
|
else
|
|
if isToken(tok, "punctuation", "(") then
|
|
tableInsert(depthStack, {startToken=tok, --[[1]]"punctuation", --[[2]]")"})
|
|
elseif isToken(tok, "punctuation", "[") then
|
|
tableInsert(depthStack, {startToken=tok, --[[1]]"punctuation", --[[2]]"]"})
|
|
elseif isToken(tok, "punctuation", "{") then
|
|
tableInsert(depthStack, {startToken=tok, --[[1]]"punctuation", --[[2]]"}"})
|
|
elseif isToken(tok, "keyword", "function") or isToken(tok, "keyword", "if") or isToken(tok, "keyword", "do") then
|
|
tableInsert(depthStack, {startToken=tok, --[[1]]"keyword", --[[2]]"end"})
|
|
elseif isToken(tok, "keyword", "repeat") then
|
|
tableInsert(depthStack, {startToken=tok, --[[1]]"keyword", --[[2]]"until"})
|
|
|
|
elseif
|
|
isToken(tok, "punctuation", ")") or
|
|
isToken(tok, "punctuation", "]") or
|
|
isToken(tok, "punctuation", "}") or
|
|
isToken(tok, "keyword", "end") or
|
|
isToken(tok, "keyword", "until")
|
|
then
|
|
if not depthStack[1] then
|
|
errorAtToken(tok, nil, "Parser/MacroArgument", "Unexpected '%s'.", tok.value)
|
|
elseif not isToken(tok, unpack(depthStack[#depthStack])) then
|
|
local startTok = depthStack[#depthStack].startToken
|
|
errorAtToken(
|
|
tok, nil, "Parser/MacroArgument", "Expected '%s' (to close '%s' %s) but got '%s'.",
|
|
depthStack[#depthStack][2], startTok.value, getRelativeLocationText(startTok, tok), tok.value
|
|
)
|
|
end
|
|
tableRemove(depthStack)
|
|
end
|
|
|
|
if not astLuaInCurrentArg then
|
|
astLuaInCurrentArg = AstLua(tok)
|
|
tableInsert(macroArg.nodes, astLuaInCurrentArg)
|
|
end
|
|
tableInsert(astLuaInCurrentArg.tokens, tok)
|
|
advanceToken(tokens) -- anything
|
|
end
|
|
end
|
|
|
|
if astLuaInCurrentArg then
|
|
-- Trim trailing useless tokens.
|
|
popUseless(astLuaInCurrentArg.tokens)
|
|
if not astLuaInCurrentArg.tokens[1] then
|
|
assert(tableRemove(macroArg.nodes) == astLuaInCurrentArg)
|
|
end
|
|
end
|
|
|
|
if not macroArg.nodes[1] and current_parsingAndMeta_strictMacroArguments then
|
|
-- There were no useful tokens for the argument!
|
|
errorAtToken(macroArg.locationToken, nil, "Parser/MacroArgument", "Expected argument #%d.", argNum)
|
|
end
|
|
|
|
-- Do next argument or finish arguments.
|
|
if isTokenAndNotNil(tokens[tokens.nextI], "punctuation", ")") then
|
|
tokens.nextI = tokens.nextI + 1 -- after ')'
|
|
break
|
|
end
|
|
|
|
assert(isToken(advanceToken(tokens), "punctuation", ",")) -- The loop above should have continued otherwise!
|
|
end--for argNum
|
|
end
|
|
|
|
-- @insert identifier !( ... ) -- Same as: @insert identifier ( !( ... ) )
|
|
-- @insert identifier !!( ... ) -- Same as: @insert identifier ( !!( ... ) )
|
|
elseif isTokenAndNotNil(tokNext, "pp_entry") then
|
|
tokens.nextI = iNext -- until '!' or '!!'
|
|
|
|
if not isTokenAndNotNil(tokens[tokens.nextI+1], "punctuation", "(") then
|
|
errorAfterToken(tokNext, "Parser/Macro", "Expected '(' after '%s'.", tokNext.value)
|
|
end
|
|
|
|
astMacro.arguments[1] = MacroArgument(tokNext, {astParseMetaBlock(tokens)}) -- The one and only argument for this macro variant.
|
|
|
|
else
|
|
errorAfterToken(astMacro.calleeTokens[#astMacro.calleeTokens], "Parser/Macro", "Expected '(' after macro name.")
|
|
end
|
|
|
|
return astMacro
|
|
end
|
|
|
|
local function astParse(params, tokens)
|
|
-- @Robustness: Make sure everywhere that key tokens came from the same source file.
|
|
local astSequence = AstSequence(tokens[1])
|
|
tokens.nextI = 1
|
|
|
|
while true do
|
|
local tok = tokens[tokens.nextI]
|
|
if not tok then break end
|
|
|
|
if isToken(tok, "pp_entry") then
|
|
tableInsert(astSequence.nodes, astParseMetaBlockOrLine(tokens))
|
|
|
|
elseif isToken(tok, "pp_keyword", "insert") then
|
|
local astMacro = astParseMacro(params, tokens)
|
|
tableInsert(astSequence.nodes, astMacro)
|
|
|
|
-- elseif isToken(tok, "pp_symbol") then -- We currently expand these in doEarlyExpansions().
|
|
-- errorAtToken(tok, nil, "Parser", "Internal error: @Incomplete: Handle symbols.")
|
|
|
|
else
|
|
local astLua = AstLua(tok)
|
|
tableInsert(astSequence.nodes, astLua)
|
|
|
|
while true do
|
|
tableInsert(astLua.tokens, tok)
|
|
advanceToken(tokens)
|
|
|
|
tok = tokens[tokens.nextI]
|
|
if not tok then break end
|
|
if tok.type:find"^pp_" then break end
|
|
end
|
|
end
|
|
end
|
|
|
|
return astSequence
|
|
end
|
|
|
|
|
|
|
|
-- lineNumber, lineNumberMeta = astNodeToMetaprogram( buffer, ast, lineNumber, lineNumberMeta, asMacroArgumentExpression )
|
|
local function astNodeToMetaprogram(buffer, ast, ln, lnMeta, asMacroArgExpr)
|
|
if current_parsingAndMeta_addLineNumbers and not asMacroArgExpr then
|
|
lnMeta = maybeOutputLineNumber(buffer, ast.locationToken, lnMeta)
|
|
end
|
|
|
|
--
|
|
-- lua -> __LUA"lua"
|
|
--
|
|
if ast.type == "lua" then
|
|
local lua = _concatTokens(ast.tokens, ln, current_parsingAndMeta_addLineNumbers, nil, nil)
|
|
ln = ast.tokens[#ast.tokens].line
|
|
|
|
if not asMacroArgExpr then tableInsert(buffer, "__LUA") end
|
|
|
|
if current_parsingAndMeta_isDebug then
|
|
if not asMacroArgExpr then tableInsert(buffer, "(") end
|
|
tableInsert(buffer, (F("%q", lua):gsub("\n", "n")))
|
|
if not asMacroArgExpr then tableInsert(buffer, ")\n") end
|
|
else
|
|
tableInsert(buffer, F("%q", lua))
|
|
if not asMacroArgExpr then tableInsert(buffer, "\n") end
|
|
end
|
|
|
|
--
|
|
-- !(expression) -> __VAL(expression)
|
|
--
|
|
elseif ast.type == "expressionValue" then
|
|
if asMacroArgExpr
|
|
then tableInsert(buffer, "__TOLUA(")
|
|
else tableInsert(buffer, "__VAL((") end
|
|
|
|
for _, tok in ipairs(ast.tokens) do
|
|
tableInsert(buffer, tok.representation)
|
|
end
|
|
|
|
if asMacroArgExpr
|
|
then tableInsert(buffer, ")")
|
|
else tableInsert(buffer, "))\n") end
|
|
|
|
--
|
|
-- !!(expression) -> __LUA(expression)
|
|
--
|
|
elseif ast.type == "expressionCode" then
|
|
if asMacroArgExpr
|
|
then tableInsert(buffer, "__ISLUA(")
|
|
else tableInsert(buffer, "__LUA((") end
|
|
|
|
for _, tok in ipairs(ast.tokens) do
|
|
tableInsert(buffer, tok.representation)
|
|
end
|
|
|
|
if asMacroArgExpr
|
|
then tableInsert(buffer, ")")
|
|
else tableInsert(buffer, "))\n") end
|
|
|
|
--
|
|
-- !(statements) -> statements
|
|
-- !statements -> statements
|
|
--
|
|
elseif ast.type == "metaprogram" then
|
|
if asMacroArgExpr then internalError(ast.type) end
|
|
|
|
if ast.originIsLine then
|
|
for i = 1, #ast.tokens-1 do
|
|
tableInsert(buffer, ast.tokens[i].representation)
|
|
end
|
|
|
|
local lastTok = ast.tokens[#ast.tokens]
|
|
if lastTok.type == "whitespace" then
|
|
if current_parsingAndMeta_isDebug
|
|
then tableInsert(buffer, (F("\n__LUA(%q)\n", lastTok.value):gsub("\\\n", "\\n"))) -- Note: "\\\n" does not match "\n".
|
|
else tableInsert(buffer, (F("\n__LUA%q\n" , lastTok.value):gsub("\\\n", "\\n"))) end
|
|
else--if type == comment
|
|
tableInsert(buffer, lastTok.representation)
|
|
if current_parsingAndMeta_isDebug
|
|
then tableInsert(buffer, F('__LUA("\\n")\n'))
|
|
else tableInsert(buffer, F("__LUA'\\n'\n" )) end
|
|
end
|
|
|
|
else
|
|
for _, tok in ipairs(ast.tokens) do
|
|
tableInsert(buffer, tok.representation)
|
|
end
|
|
tableInsert(buffer, "\n")
|
|
end
|
|
|
|
--
|
|
-- @@callee(argument1, ...) -> __LUA(__M(callee(__ARG(1,<argument1>), ...)))
|
|
-- OR -> __LUA(__M(callee(__ARG(1,function()<argument1>end), ...)))
|
|
--
|
|
-- The code handling each argument will be different depending on the complexity of the argument.
|
|
--
|
|
elseif ast.type == "macro" then
|
|
if not asMacroArgExpr then tableInsert(buffer, "__LUA(") end
|
|
|
|
tableInsert(buffer, "__M()(")
|
|
for _, tok in ipairs(ast.calleeTokens) do
|
|
tableInsert(buffer, tok.representation)
|
|
end
|
|
tableInsert(buffer, "(")
|
|
|
|
for argNum, macroArg in ipairs(ast.arguments) do
|
|
local argIsComplex = false -- If any part of the argument cannot be an expression then it's complex.
|
|
|
|
for _, astInArg in ipairs(macroArg.nodes) do
|
|
if astInArg.type == "metaprogram" or astInArg.type == "dualCode" then
|
|
argIsComplex = true
|
|
break
|
|
end
|
|
end
|
|
|
|
if argNum > 1 then
|
|
tableInsert(buffer, ",")
|
|
if current_parsingAndMeta_isDebug then tableInsert(buffer, " ") end
|
|
end
|
|
|
|
local locTokNum = #current_meta_locationTokens + 1
|
|
current_meta_locationTokens[locTokNum] = macroArg.nodes[1] and macroArg.nodes[1].locationToken or macroArg.locationToken or internalError()
|
|
|
|
tableInsert(buffer, "__ARG(")
|
|
tableInsert(buffer, tostring(locTokNum))
|
|
tableInsert(buffer, ",")
|
|
|
|
if argIsComplex then
|
|
tableInsert(buffer, "function()\n")
|
|
for nodeNumInArg, astInArg in ipairs(macroArg.nodes) do
|
|
ln, lnMeta = astNodeToMetaprogram(buffer, astInArg, ln, lnMeta, false)
|
|
end
|
|
tableInsert(buffer, "end")
|
|
|
|
elseif macroArg.nodes[1] then
|
|
for nodeNumInArg, astInArg in ipairs(macroArg.nodes) do
|
|
if nodeNumInArg > 1 then tableInsert(buffer, "..") end
|
|
ln, lnMeta = astNodeToMetaprogram(buffer, astInArg, ln, lnMeta, true)
|
|
end
|
|
|
|
else
|
|
tableInsert(buffer, '""')
|
|
end
|
|
|
|
tableInsert(buffer, ")")
|
|
end
|
|
|
|
tableInsert(buffer, "))")
|
|
|
|
if not asMacroArgExpr then tableInsert(buffer, ")\n") end
|
|
|
|
--
|
|
-- !!local names = values -> local names = values ; __LUA"local names = "__VAL(name1)__LUA", "__VAL(name2)...
|
|
-- !! names = values -> names = values ; __LUA"names = "__VAL(name1)__LUA", "__VAL(name2)...
|
|
--
|
|
elseif ast.type == "dualCode" then
|
|
if asMacroArgExpr then internalError(ast.type) end
|
|
|
|
-- Metaprogram.
|
|
if ast.isDeclaration then tableInsert(buffer, "local ") end
|
|
tableInsert(buffer, table.concat(ast.names, ", "))
|
|
tableInsert(buffer, ' = ')
|
|
for _, tok in ipairs(ast.valueTokens) do
|
|
tableInsert(buffer, tok.representation)
|
|
end
|
|
|
|
-- Final program.
|
|
tableInsert(buffer, '__LUA')
|
|
if current_parsingAndMeta_isDebug then tableInsert(buffer, '(') end
|
|
tableInsert(buffer, '"') -- string start
|
|
if current_parsingAndMeta_addLineNumbers then
|
|
ln = maybeOutputLineNumber(buffer, ast.locationToken, ln)
|
|
end
|
|
if ast.isDeclaration then tableInsert(buffer, "local ") end
|
|
tableInsert(buffer, table.concat(ast.names, ", "))
|
|
tableInsert(buffer, ' = "') -- string end
|
|
if current_parsingAndMeta_isDebug then tableInsert(buffer, '); ') end
|
|
|
|
for i, name in ipairs(ast.names) do
|
|
if i == 1 then -- void
|
|
elseif current_parsingAndMeta_isDebug then tableInsert(buffer, '; __LUA(", "); ')
|
|
else tableInsert(buffer, '__LUA", "' ) end
|
|
tableInsert(buffer, "__VAL(")
|
|
tableInsert(buffer, name)
|
|
tableInsert(buffer, ")")
|
|
end
|
|
|
|
-- Use trailing semicolon if the user does.
|
|
for i = #ast.valueTokens, 1, -1 do
|
|
if isToken(ast.valueTokens[i], "punctuation", ";") then
|
|
if current_parsingAndMeta_isDebug
|
|
then tableInsert(buffer, '; __LUA(";")')
|
|
else tableInsert(buffer, '__LUA";"' ) end
|
|
break
|
|
elseif not isToken(ast.valueTokens[i], "whitespace") then
|
|
break
|
|
end
|
|
end
|
|
|
|
if current_parsingAndMeta_isDebug
|
|
then tableInsert(buffer, '; __LUA("\\n")\n')
|
|
else tableInsert(buffer, '__LUA"\\n"\n' ) end
|
|
|
|
--
|
|
-- ...
|
|
--
|
|
elseif ast.type == "sequence" then
|
|
for _, astChild in ipairs(ast.nodes) do
|
|
ln, lnMeta = astNodeToMetaprogram(buffer, astChild, ln, lnMeta, false)
|
|
end
|
|
|
|
-- elseif ast.type == "symbol" then
|
|
-- errorAtToken(ast.locationToken, nil, nil, "AstSymbol")
|
|
|
|
else
|
|
printErrorTraceback("Internal error.")
|
|
errorAtToken(ast.locationToken, nil, "Parsing", "Internal error. (%s, %s)", ast.type, tostring(asMacroArgExpr))
|
|
end
|
|
|
|
return ln, lnMeta
|
|
end
|
|
|
|
local function astToLua(ast)
|
|
local buffer = {}
|
|
astNodeToMetaprogram(buffer, ast, 0, 0, false)
|
|
return table.concat(buffer)
|
|
end
|
|
|
|
|
|
|
|
local function _processFileOrString(params, isFile)
|
|
if isFile then
|
|
if not params.pathIn then error("Missing 'pathIn' in params.", 2) end
|
|
if not params.pathOut then error("Missing 'pathOut' in params.", 2) end
|
|
|
|
if params.pathOut == params.pathIn and params.pathOut ~= "-" then
|
|
error("'pathIn' and 'pathOut' are the same in params.", 2)
|
|
end
|
|
|
|
if (params.pathMeta or "-") == "-" then -- Should it be possible to output the metaprogram to stdout?
|
|
-- void
|
|
elseif params.pathMeta == params.pathIn then
|
|
error("'pathIn' and 'pathMeta' are the same in params.", 2)
|
|
elseif params.pathMeta == params.pathOut then
|
|
error("'pathOut' and 'pathMeta' are the same in params.", 2)
|
|
end
|
|
|
|
else
|
|
if not params.code then error("Missing 'code' in params.", 2) end
|
|
end
|
|
|
|
-- Read input.
|
|
local luaUnprocessed, virtualPathIn
|
|
|
|
if isFile then
|
|
virtualPathIn = params.pathIn
|
|
local err
|
|
|
|
if virtualPathIn == "-" then
|
|
luaUnprocessed, err = io.stdin:read"*a"
|
|
else
|
|
luaUnprocessed, err = readFile(virtualPathIn, true)
|
|
end
|
|
|
|
if not luaUnprocessed then
|
|
errorf("Could not read file '%s'. (%s)", virtualPathIn, err)
|
|
end
|
|
|
|
current_anytime_pathIn = params.pathIn
|
|
current_anytime_pathOut = params.pathOut
|
|
|
|
else
|
|
virtualPathIn = "<code>"
|
|
luaUnprocessed = params.code
|
|
end
|
|
|
|
current_anytime_fastStrings = params.fastStrings
|
|
current_parsing_insertCount = 0
|
|
current_parsingAndMeta_resourceCache = {[virtualPathIn]=luaUnprocessed} -- The contents of files, unless params.onInsert() is specified in which case it's user defined.
|
|
current_parsingAndMeta_onInsert = params.onInsert
|
|
current_parsingAndMeta_addLineNumbers = params.addLineNumbers
|
|
current_parsingAndMeta_macroPrefix = params.macroPrefix or ""
|
|
current_parsingAndMeta_macroSuffix = params.macroSuffix or ""
|
|
current_parsingAndMeta_strictMacroArguments = params.strictMacroArguments ~= false
|
|
current_meta_locationTokens = {}
|
|
|
|
local specialFirstLine, rest = luaUnprocessed:match"^(#[^\r\n]*\r?\n?)(.*)$"
|
|
if specialFirstLine then
|
|
specialFirstLine = specialFirstLine:gsub("\r", "") -- Normalize line breaks. (Assume the input is either "\n" or "\r\n".)
|
|
luaUnprocessed = rest
|
|
end
|
|
|
|
-- Ensure there's a newline at the end of the code, otherwise there will be problems down the line.
|
|
if not (luaUnprocessed == "" or luaUnprocessed:find"\n%s*$") then
|
|
luaUnprocessed = luaUnprocessed .. "\n"
|
|
end
|
|
|
|
local tokens = _tokenize(luaUnprocessed, virtualPathIn, true, params.backtickStrings, params.jitSyntax)
|
|
-- printTokens(tokens) -- DEBUG
|
|
|
|
-- Gather info.
|
|
local lastTok = tokens[#tokens]
|
|
|
|
local stats = {
|
|
processedByteCount = #luaUnprocessed,
|
|
lineCount = (specialFirstLine and 1 or 0) + (lastTok and lastTok.line + countString(lastTok.representation, "\n", true) or 0),
|
|
lineCountCode = getLineCountWithCode(tokens),
|
|
tokenCount = 0, -- Set later.
|
|
hasPreprocessorCode = false,
|
|
hasMetaprogram = false,
|
|
insertedNames = {},
|
|
}
|
|
|
|
for _, tok in ipairs(tokens) do
|
|
-- @Volatile: Make sure to update this when syntax is changed!
|
|
if isToken(tok, "pp_entry") or isToken(tok, "pp_keyword", "insert") or isToken(tok, "pp_symbol") then
|
|
stats.hasPreprocessorCode = true
|
|
stats.hasMetaprogram = true
|
|
break
|
|
elseif isToken(tok, "pp_keyword") or (isToken(tok, "string") and tok.representation:find"^`") then
|
|
stats.hasPreprocessorCode = true
|
|
-- Keep going as there may be metaprogram.
|
|
end
|
|
end
|
|
|
|
-- Generate and run metaprogram.
|
|
----------------------------------------------------------------
|
|
|
|
local shouldProcess = stats.hasPreprocessorCode or params.addLineNumbers
|
|
|
|
if shouldProcess then
|
|
tokens = doExpansions(params, tokens, stats)
|
|
end
|
|
stats.tokenCount = #tokens
|
|
|
|
current_meta_maxLogLevel = params.logLevel or "trace"
|
|
if not LOG_LEVELS[current_meta_maxLogLevel] then
|
|
errorf(2, "Invalid 'logLevel' value in params. (%s)", tostring(current_meta_maxLogLevel))
|
|
end
|
|
|
|
local lua
|
|
|
|
if shouldProcess then
|
|
local luaMeta = astToLua(astParse(params, tokens))
|
|
--[[ DEBUG :PrintCode
|
|
print("=META===============================")
|
|
print(luaMeta)
|
|
print("====================================")
|
|
--]]
|
|
|
|
-- Run metaprogram.
|
|
current_meta_pathForErrorMessages = params.pathMeta or "<meta>"
|
|
current_meta_output = {}
|
|
current_meta_outputStack = {current_meta_output}
|
|
current_meta_canOutputNil = params.canOutputNil ~= false
|
|
current_meta_releaseMode = params.release
|
|
|
|
if params.pathMeta then
|
|
local file, err = io.open(params.pathMeta, "wb")
|
|
if not file then errorf("Count not open '%s' for writing. (%s)", params.pathMeta, err) end
|
|
|
|
file:write(luaMeta)
|
|
file:close()
|
|
end
|
|
|
|
if params.onBeforeMeta then params.onBeforeMeta(luaMeta) end
|
|
|
|
local main_chunk, err = loadLuaString(luaMeta, "@"..current_meta_pathForErrorMessages, metaEnv)
|
|
if not main_chunk then
|
|
local ln, _err = err:match"^.-:(%d+): (.*)"
|
|
errorOnLine(current_meta_pathForErrorMessages, (tonumber(ln) or 0), nil, "%s", (_err or err))
|
|
end
|
|
|
|
current_anytime_isRunningMeta = true
|
|
main_chunk() -- Note: Our caller should clean up current_meta_pathForErrorMessages etc. on error.
|
|
current_anytime_isRunningMeta = false
|
|
|
|
if not current_parsingAndMeta_isDebug and params.pathMeta then
|
|
os.remove(params.pathMeta)
|
|
end
|
|
|
|
if current_meta_outputStack[2] then
|
|
error("Called startInterceptingOutput() more times than stopInterceptingOutput().")
|
|
end
|
|
|
|
lua = table.concat(current_meta_output)
|
|
--[[ DEBUG :PrintCode
|
|
print("=OUTPUT=============================")
|
|
print(lua)
|
|
print("====================================")
|
|
--]]
|
|
|
|
current_meta_pathForErrorMessages = ""
|
|
current_meta_output = nil
|
|
current_meta_outputStack = nil
|
|
current_meta_canOutputNil = true
|
|
current_meta_releaseMode = false
|
|
|
|
else
|
|
-- @Copypaste from above.
|
|
if not current_parsingAndMeta_isDebug and params.pathMeta then
|
|
os.remove(params.pathMeta)
|
|
end
|
|
|
|
lua = luaUnprocessed
|
|
end
|
|
|
|
current_meta_maxLogLevel = "trace"
|
|
current_meta_locationTokens = nil
|
|
|
|
if params.onAfterMeta then
|
|
local luaModified = params.onAfterMeta(lua)
|
|
|
|
if type(luaModified) == "string" then
|
|
lua = luaModified
|
|
elseif luaModified ~= nil then
|
|
errorf("onAfterMeta() did not return a string. (Got %s)", type(luaModified))
|
|
end
|
|
end
|
|
|
|
-- Write output file.
|
|
----------------------------------------------------------------
|
|
|
|
local pathOut = isFile and params.pathOut or "<output>"
|
|
|
|
if isFile then
|
|
if pathOut == "-" then
|
|
io.stdout:write(specialFirstLine or "")
|
|
io.stdout:write(lua)
|
|
|
|
else
|
|
local file, err = io.open(pathOut, "wb")
|
|
if not file then errorf("Count not open '%s' for writing. (%s)", pathOut, err) end
|
|
|
|
file:write(specialFirstLine or "")
|
|
file:write(lua)
|
|
file:close()
|
|
end
|
|
end
|
|
|
|
-- Check if the output is valid Lua.
|
|
if params.validate ~= false then
|
|
local luaToCheck = lua:gsub("^#![^\n]*", "")
|
|
local chunk, err = loadLuaString(luaToCheck, "@"..pathOut, nil)
|
|
|
|
if not chunk then
|
|
local ln, _err = err:match"^.-:(%d+): (.*)"
|
|
errorOnLine(pathOut, (tonumber(ln) or 0), nil, "Output is invalid Lua. (%s)", (_err or err))
|
|
end
|
|
end
|
|
|
|
-- :ProcessInfo
|
|
local info = {
|
|
path = isFile and params.pathIn or "",
|
|
outputPath = isFile and params.pathOut or "",
|
|
processedByteCount = stats.processedByteCount,
|
|
lineCount = stats.lineCount,
|
|
linesOfCode = stats.lineCountCode,
|
|
tokenCount = stats.tokenCount,
|
|
hasPreprocessorCode = stats.hasPreprocessorCode,
|
|
hasMetaprogram = stats.hasMetaprogram,
|
|
insertedFiles = stats.insertedNames,
|
|
}
|
|
|
|
if params.onDone then params.onDone(info) end
|
|
|
|
current_anytime_pathIn = ""
|
|
current_anytime_pathOut = ""
|
|
current_anytime_fastStrings = false
|
|
current_parsingAndMeta_resourceCache = nil
|
|
current_parsingAndMeta_onInsert = nil
|
|
current_parsingAndMeta_addLineNumbers = false
|
|
current_parsingAndMeta_macroPrefix = ""
|
|
current_parsingAndMeta_macroSuffix = ""
|
|
current_parsingAndMeta_strictMacroArguments = true
|
|
|
|
----------------------------------------------------------------
|
|
|
|
if isFile then
|
|
return info
|
|
else
|
|
if specialFirstLine then
|
|
lua = specialFirstLine .. lua
|
|
end
|
|
return lua, info
|
|
end
|
|
end
|
|
|
|
local function processFileOrString(params, isFile)
|
|
if current_parsingAndMeta_isProcessing then
|
|
error("Cannot process recursively.", 3) -- Note: We don't return failure in this case - it's a critical error!
|
|
end
|
|
|
|
-- local startTime = os.clock() -- :DebugMeasureTime @Incomplete: Add processing time to returned info.
|
|
local returnValues = nil
|
|
|
|
current_parsingAndMeta_isProcessing = true
|
|
current_parsingAndMeta_isDebug = params.debug
|
|
|
|
local xpcallOk, xpcallErr = xpcall(
|
|
function()
|
|
returnValues = pack(_processFileOrString(params, isFile))
|
|
end,
|
|
|
|
function(err)
|
|
if type(err) == "string" and err:find("\0", 1, true) then
|
|
printError(tryToFormatError(cleanError(err)))
|
|
else
|
|
printErrorTraceback(err, 2) -- The level should be at error().
|
|
end
|
|
|
|
if params.onError then
|
|
local cbOk, cbErr = pcall(params.onError, err)
|
|
if not cbOk then
|
|
printfError("Additional error in params.onError()...\n%s", tryToFormatError(cbErr))
|
|
end
|
|
end
|
|
|
|
return err
|
|
end
|
|
)
|
|
|
|
current_parsingAndMeta_isProcessing = false
|
|
current_parsingAndMeta_isDebug = false
|
|
|
|
-- Cleanup in case an error happened.
|
|
current_anytime_isRunningMeta = false
|
|
current_anytime_pathIn = ""
|
|
current_anytime_pathOut = ""
|
|
current_anytime_fastStrings = false
|
|
current_parsing_insertCount = 0
|
|
current_parsingAndMeta_onInsert = nil
|
|
current_parsingAndMeta_resourceCache = nil
|
|
current_parsingAndMeta_addLineNumbers = false
|
|
current_parsingAndMeta_macroPrefix = ""
|
|
current_parsingAndMeta_macroSuffix = ""
|
|
current_parsingAndMeta_strictMacroArguments = true
|
|
current_meta_pathForErrorMessages = ""
|
|
current_meta_output = nil
|
|
current_meta_outputStack = nil
|
|
current_meta_canOutputNil = true
|
|
current_meta_releaseMode = false
|
|
current_meta_maxLogLevel = "trace"
|
|
current_meta_locationTokens = nil
|
|
|
|
-- print("time", os.clock()-startTime) -- :DebugMeasureTime
|
|
if xpcallOk then
|
|
return unpack(returnValues, 1, returnValues.n)
|
|
else
|
|
return nil, cleanError(xpcallErr or "Unknown processing error.")
|
|
end
|
|
end
|
|
|
|
local function processFile(params)
|
|
local returnValues = pack(processFileOrString(params, true))
|
|
return unpack(returnValues, 1, returnValues.n)
|
|
end
|
|
|
|
local function processString(params)
|
|
local returnValues = pack(processFileOrString(params, false))
|
|
return unpack(returnValues, 1, returnValues.n)
|
|
end
|
|
|
|
|
|
|
|
-- :ExportTable
|
|
local pp = {
|
|
|
|
-- Processing functions.
|
|
----------------------------------------------------------------
|
|
|
|
-- processFile()
|
|
-- Process a Lua file. Returns nil and a message on error.
|
|
--
|
|
-- info = processFile( params )
|
|
-- info: Table with various information. (See 'ProcessInfo' for more info.)
|
|
--
|
|
-- params: Table with these fields:
|
|
-- pathIn = pathToInputFile -- [Required] Specify "-" to use stdin.
|
|
-- pathOut = pathToOutputFile -- [Required] Specify "-" to use stdout. (Note that if stdout is used then anything you print() in the metaprogram will end up there.)
|
|
-- pathMeta = pathForMetaprogram -- [Optional] You can inspect this temporary output file if an error occurs in the metaprogram.
|
|
--
|
|
-- debug = boolean -- [Optional] Debug mode. The metaprogram file is formatted more nicely and does not get deleted automatically.
|
|
-- addLineNumbers = boolean -- [Optional] Add comments with line numbers to the output.
|
|
--
|
|
-- backtickStrings = boolean -- [Optional] Enable the backtick (`) to be used as string literal delimiters. Backtick strings don't interpret any escape sequences and can't contain other backticks. (Default: false)
|
|
-- jitSyntax = boolean -- [Optional] Allow LuaJIT-specific syntax. (Default: false)
|
|
-- canOutputNil = boolean -- [Optional] Allow !(expression) and outputValue() to output nil. (Default: true)
|
|
-- fastStrings = boolean -- [Optional] Force fast serialization of string values. (Non-ASCII characters will look ugly.) (Default: false)
|
|
-- validate = boolean -- [Optional] Validate output. (Default: true)
|
|
-- strictMacroArguments = boolean -- [Optional] Check that macro arguments are valid Lua expressions. (Default: true)
|
|
--
|
|
-- macroPrefix = prefix -- [Optional] String to prepend to macro names. (Default: "")
|
|
-- macroSuffix = suffix -- [Optional] String to append to macro names. (Default: "")
|
|
--
|
|
-- release = boolean -- [Optional] Enable release mode. Currently only disables the @@ASSERT() macro when true. (Default: false)
|
|
-- logLevel = levelName -- [Optional] Maximum log level for the @@LOG() macro. Can be "off", "error", "warning", "info", "debug" or "trace". (Default: "trace", which enables all logging)
|
|
--
|
|
-- onInsert = function( name ) -- [Optional] Called for each @insert"name" instruction. It's expected to return a Lua code string. By default 'name' is a path to a file to be inserted.
|
|
-- onBeforeMeta = function( luaString ) -- [Optional] Called before the metaprogram runs, if a metaprogram is generated. luaString contains the metaprogram.
|
|
-- onAfterMeta = function( luaString ) -- [Optional] Here you can modify and return the Lua code before it's written to 'pathOut'.
|
|
-- onError = function( error ) -- [Optional] You can use this to get traceback information. 'error' is the same value as what is returned from processFile().
|
|
--
|
|
processFile = processFile,
|
|
|
|
-- processString()
|
|
-- Process Lua code. Returns nil and a message on error.
|
|
--
|
|
-- luaString, info = processString( params )
|
|
-- info: Table with various information. (See 'ProcessInfo' for more info.)
|
|
--
|
|
-- params: Table with these fields:
|
|
-- code = luaString -- [Required]
|
|
-- pathMeta = pathForMetaprogram -- [Optional] You can inspect this temporary output file if an error occurs in the metaprogram.
|
|
--
|
|
-- debug = boolean -- [Optional] Debug mode. The metaprogram file is formatted more nicely and does not get deleted automatically.
|
|
-- addLineNumbers = boolean -- [Optional] Add comments with line numbers to the output.
|
|
--
|
|
-- backtickStrings = boolean -- [Optional] Enable the backtick (`) to be used as string literal delimiters. Backtick strings don't interpret any escape sequences and can't contain other backticks. (Default: false)
|
|
-- jitSyntax = boolean -- [Optional] Allow LuaJIT-specific syntax. (Default: false)
|
|
-- canOutputNil = boolean -- [Optional] Allow !(expression) and outputValue() to output nil. (Default: true)
|
|
-- fastStrings = boolean -- [Optional] Force fast serialization of string values. (Non-ASCII characters will look ugly.) (Default: false)
|
|
-- validate = boolean -- [Optional] Validate output. (Default: true)
|
|
-- strictMacroArguments = boolean -- [Optional] Check that macro arguments are valid Lua expressions. (Default: true)
|
|
--
|
|
-- macroPrefix = prefix -- [Optional] String to prepend to macro names. (Default: "")
|
|
-- macroSuffix = suffix -- [Optional] String to append to macro names. (Default: "")
|
|
--
|
|
-- release = boolean -- [Optional] Enable release mode. Currently only disables the @@ASSERT() macro when true. (Default: false)
|
|
-- logLevel = levelName -- [Optional] Maximum log level for the @@LOG() macro. Can be "off", "error", "warning", "info", "debug" or "trace". (Default: "trace", which enables all logging)
|
|
--
|
|
-- onInsert = function( name ) -- [Optional] Called for each @insert"name" instruction. It's expected to return a Lua code string. By default 'name' is a path to a file to be inserted.
|
|
-- onBeforeMeta = function( luaString ) -- [Optional] Called before the metaprogram runs, if a metaprogram is generated. luaString contains the metaprogram.
|
|
-- onError = function( error ) -- [Optional] You can use this to get traceback information. 'error' is the same value as the second returned value from processString().
|
|
--
|
|
processString = processString,
|
|
|
|
-- Values.
|
|
----------------------------------------------------------------
|
|
|
|
VERSION = PP_VERSION, -- The version of LuaPreprocess.
|
|
metaEnvironment = metaEnv, -- The environment used for metaprograms.
|
|
}
|
|
|
|
-- Include all functions from the metaprogram environment.
|
|
for k, v in pairs(metaFuncs) do pp[k] = v end
|
|
|
|
return pp
|
|
|
|
|
|
|
|
--[[!===========================================================
|
|
|
|
Copyright © 2018-2022 Marcus 'ReFreezed' Thunström
|
|
|
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
of this software and associated documentation files (the "Software"), to deal
|
|
in the Software without restriction, including without limitation the rights
|
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
copies of the Software, and to permit persons to whom the Software is
|
|
furnished to do so, subject to the following conditions:
|
|
|
|
The above copyright notice and this permission notice shall be included in all
|
|
copies or substantial portions of the Software.
|
|
|
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
SOFTWARE.
|
|
|
|
==============================================================]]
|
|
|