Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
openSUSE:Evergreen:11.2
texlive
texmf-luatools.dif
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File texmf-luatools.dif of Package texlive
--- texmf-dist/scripts/context/lua/luatools.lua +++ texmf-dist/scripts/context/lua/luatools.lua 2008-10-30 14:05:11.270545206 +0100 @@ -3,19 +3,21 @@ -- one can make a stub: -- -- #!/bin/sh --- env LUATEXDIR=/....../texmf/scripts/context/lua luatex --luaonly=luatools.lua "$@" +-- env LUATEXDIR=/....../texmf/scripts/context/lua texlua luatools.lua "$@" + -- filename : luatools.lua -- comment : companion to context.tex -- author : Hans Hagen, PRAGMA-ADE, Hasselt NL -- copyright: PRAGMA ADE / ConTeXt Development Team -- license : see context related readme files + -- Although this script is part of the ConTeXt distribution it is -- relatively indepent of ConTeXt. The same is true for some of -- the luat files. We may may make them even less dependent in -- the future. As long as Luatex is under development the -- interfaces and names of functions may change. -banner = "version 1.2.0 - 2006+ - PRAGMA ADE / CONTEXT" +banner = "version 1.2.2 - 2006+ - PRAGMA ADE / CONTEXT" texlua = true -- For the sake of independence we optionally can merge the library @@ -111,7 +113,7 @@ end function string:splitchr(chr) if #self > 0 then local t = { } - for s in string.gmatch(self..chr,"(.-)"..chr) do + for s in (self..chr):gmatch("(.-)"..chr) do t[#t+1] = s end return t @@ -120,22 +122,6 @@ function string:splitchr(chr) end end ---~ function string.piecewise(str, pat, fnc) -- variant of split ---~ local fpat = "(.-)"..pat ---~ local last_end = 1 ---~ local s, e, cap = string.find(str, fpat, 1) ---~ while s ~= nil do ---~ if s~=1 or cap~="" then ---~ fnc(cap) ---~ end ---~ last_end = e+1 ---~ s, e, cap = string.find(str, fpat, last_end) ---~ end ---~ if last_end <= #str then ---~ fnc((string.sub(str,last_end))) ---~ end ---~ end - function string.piecewise(str, pat, fnc) -- variant of split for k in string.splitter(str,pat) do fnc(k) end end @@ -177,7 +163,7 @@ end --~ end end -string.chr_to_esc = { +local chr_to_esc = { ["%"] = "%%", ["."] = "%.", ["+"] = "%+", ["-"] = "%-", ["*"] = "%*", @@ -187,16 +173,18 @@ string.chr_to_esc = { ["{"] = "%{", ["}"] = "%}" } +string.chr_to_esc = chr_to_esc + function string:esc() -- variant 2 - return (self:gsub("(.)",string.chr_to_esc)) + return (self:gsub("(.)",chr_to_esc)) end -function string.unquote(str) - return (str:gsub("^([\"\'])(.*)%1$","%2")) +function string:unquote() + return (self:gsub("^([\"\'])(.*)%1$","%2")) end -function string.quote(str) - return '"' .. str:unquote() .. '"' +function string:quote() + return '"' .. self:unquote() .. '"' end function string:count(pattern) -- variant 3 @@ -446,6 +434,30 @@ function string:splitlines() return capture:match(self) end +--~ local p = lpeg.splitat("->",false) print(p:match("oeps->what->more")) -- oeps what more +--~ local p = lpeg.splitat("->",true) print(p:match("oeps->what->more")) -- oeps what->more +--~ local p = lpeg.splitat("->",false) print(p:match("oeps")) -- oeps +--~ local p = lpeg.splitat("->",true) print(p:match("oeps")) -- oeps + +local splitters_s, splitters_m = { }, { } + +function lpeg.splitat(separator,single) + local splitter = (single and splitters_s[separator]) or splitters_m[separator] + if not splitter then + separator = lpeg.P(separator) + if single then + local other, any = lpeg.C((1 - separator)^0), lpeg.P(1) + splitter = other * (separator * lpeg.C(any^0) + "") + splitters_s[separator] = splitter + else + local other = lpeg.C((1 - separator)^0) + splitter = other * (separator * other)^0 + splitters_m[separator] = splitter + end + end + return splitter +end + -- filename : l-table.lua -- comment : split off from luat-lib @@ -457,11 +469,15 @@ if not versions then versions = { } end table.join = table.concat +local concat, sort, insert, remove = table.concat, table.sort, table.insert, table.remove +local format = string.format +local getmetatable, setmetatable = getmetatable, setmetatable +local pairs, ipairs, type, next, tostring = pairs, ipairs, type, next, tostring + function table.strip(tab) local lst = { } - for k, v in ipairs(tab) do - -- s = string.gsub(v, "^%s*(.-)%s*$", "%1") - s = v:gsub("^%s*(.-)%s*$", "%1") + for i=1,#tab do + local s = tab[i]:gsub("^%s*(.-)%s*$","%1") if s == "" then -- skip this one else @@ -471,16 +487,7 @@ function table.strip(tab) return lst end ---~ function table.sortedkeys(tab) ---~ local srt = { } ---~ for key,_ in pairs(tab) do ---~ srt[#srt+1] = key ---~ end ---~ table.sort(srt) ---~ return srt ---~ end - -function table.sortedkeys(tab) +local function sortedkeys(tab) local srt, kind = { }, 0 -- 0=unknown 1=string, 2=number 3=mixed for key,_ in pairs(tab) do srt[#srt+1] = key @@ -500,22 +507,34 @@ function table.sortedkeys(tab) end end if kind == 0 or kind == 3 then - table.sort(srt,function(a,b) return (tostring(a) < tostring(b)) end) + sort(srt,function(a,b) return (tostring(a) < tostring(b)) end) else - table.sort(srt) + sort(srt) + end + return srt +end + +local function sortedhashkeys(tab) -- fast one + local srt = { } + for key,_ in pairs(tab) do + srt[#srt+1] = key end + sort(srt) return srt end +table.sortedkeys = sortedkeys +table.sortedhashkeys = sortedhashkeys + function table.append(t, list) for _,v in pairs(list) do - table.insert(t,v) + insert(t,v) end end function table.prepend(t, list) for k,v in pairs(list) do - table.insert(t,k,v) + insert(t,k,v) end end @@ -562,70 +581,57 @@ function table.imerged(...) return tmp end -if not table.fastcopy then do - - local type, pairs, getmetatable, setmetatable = type, pairs, getmetatable, setmetatable - - local function fastcopy(old) -- fast one - if old then - local new = { } - for k,v in pairs(old) do - if type(v) == "table" then - new[k] = fastcopy(v) -- was just table.copy - else - new[k] = v - end - end - local mt = getmetatable(old) - if mt then - setmetatable(new,mt) +local function fastcopy(old) -- fast one + if old then + local new = { } + for k,v in pairs(old) do + if type(v) == "table" then + new[k] = fastcopy(v) -- was just table.copy + else + new[k] = v end - return new - else - return { } end + local mt = getmetatable(old) + if mt then + setmetatable(new,mt) + end + return new + else + return { } end +end - table.fastcopy = fastcopy - -end end - -if not table.copy then do - - local type, pairs, getmetatable, setmetatable = type, pairs, getmetatable, setmetatable - - local function copy(t, tables) -- taken from lua wiki, slightly adapted - tables = tables or { } - local tcopy = {} - if not tables[t] then - tables[t] = tcopy - end - for i,v in pairs(t) do -- brrr, what happens with sparse indexed - if type(i) == "table" then - if tables[i] then - i = tables[i] - else - i = copy(i, tables) - end - end - if type(v) ~= "table" then - tcopy[i] = v - elseif tables[v] then - tcopy[i] = tables[v] +local function copy(t, tables) -- taken from lua wiki, slightly adapted + tables = tables or { } + local tcopy = {} + if not tables[t] then + tables[t] = tcopy + end + for i,v in pairs(t) do -- brrr, what happens with sparse indexed + if type(i) == "table" then + if tables[i] then + i = tables[i] else - tcopy[i] = copy(v, tables) + i = copy(i, tables) end end - local mt = getmetatable(t) - if mt then - setmetatable(tcopy,mt) + if type(v) ~= "table" then + tcopy[i] = v + elseif tables[v] then + tcopy[i] = tables[v] + else + tcopy[i] = copy(v, tables) end - return tcopy end + local mt = getmetatable(t) + if mt then + setmetatable(tcopy,mt) + end + return tcopy +end - table.copy = copy - -end end +table.fastcopy = fastcopy +table.copy = copy -- rougly: copy-loop : unpack : sub == 0.9 : 0.4 : 0.45 (so in critical apps, use unpack) @@ -654,334 +660,363 @@ function table.starts_at(t) return ipairs(t,1)(t,0) end -do +function table.tohash(t,value) + local h = { } + if value == nil then value = true end + for _, v in pairs(t) do -- no ipairs here + h[v] = value + end + return h +end + +function table.fromhash(t) + local h = { } + for k, v in pairs(t) do -- no ipairs here + if v then h[#h+1] = k end + end + return h +end - -- one of my first exercises in lua ... +--~ print(table.serialize(t), "\n") +--~ print(table.serialize(t,"name"), "\n") +--~ print(table.serialize(t,false), "\n") +--~ print(table.serialize(t,true), "\n") +--~ print(table.serialize(t,"name",true), "\n") +--~ print(table.serialize(t,"name",true,true), "\n") - -- 34.055.092 32.403.326 arabtype.tma - -- 1.620.614 1.513.863 lmroman10-italic.tma - -- 1.325.585 1.233.044 lmroman10-regular.tma - -- 1.248.157 1.158.903 lmsans10-regular.tma - -- 194.646 153.120 lmtypewriter10-regular.tma - -- 1.771.678 1.658.461 palatinosanscom-bold.tma - -- 1.695.251 1.584.491 palatinosanscom-regular.tma - -- 13.736.534 13.409.446 zapfinoextraltpro.tma - - -- 13.679.038 11.774.106 arabtype.tmc - -- 886.248 754.944 lmroman10-italic.tmc - -- 729.828 466.864 lmroman10-regular.tmc - -- 688.482 441.962 lmsans10-regular.tmc - -- 128.685 95.853 lmtypewriter10-regular.tmc - -- 715.929 582.985 palatinosanscom-bold.tmc - -- 669.942 540.126 palatinosanscom-regular.tmc - -- 1.560.588 1.317.000 zapfinoextraltpro.tmc - - table.serialize_functions = true - table.serialize_compact = true - table.serialize_inline = true +table.serialize_functions = true +table.serialize_compact = true +table.serialize_inline = true + +local noquotes, hexify, handle, reduce, compact, inline, functions + +local reserved = table.tohash { -- intercept a language flaw, no reserved words as key + 'and', 'break', 'do', 'else', 'elseif', 'end', 'false', 'for', 'function', 'if', + 'in', 'local', 'nil', 'not', 'or', 'repeat', 'return', 'then', 'true', 'until', 'while', +} - local function key(k) - if type(k) == "number" then -- or k:find("^%d+$") then - return "["..k.."]" - elseif noquotes and k:find("^%a[%a%d%_]*$") then - return k +local function key(k) + if type(k) == "number" then -- or k:find("^%d+$") then + if hexify then + return ("[0x%04X]"):format(k) else - return '["'..k..'"]' + return "["..k.."]" end + elseif noquotes and not reserved[k] and k:find("^%a[%a%d%_]*$") then + return k + else + return '["'..k..'"]' end +end - local function simple_table(t) - if #t > 0 then - local n = 0 - for _,v in pairs(t) do - n = n + 1 - end - if n == #t then - local tt = { } - for i=1,#t do - local v = t[i] - local tv = type(v) - if tv == "number" or tv == "boolean" then - tt[#tt+1] = tostring(v) - elseif tv == "string" then - tt[#tt+1] = ("%q"):format(v) +local function simple_table(t) + if #t > 0 then + local n = 0 + for _,v in pairs(t) do + n = n + 1 + end + if n == #t then + local tt = { } + for i=1,#t do + local v = t[i] + local tv = type(v) + if tv == "number" then + if hexify then + tt[#tt+1] = ("0x%04X"):format(v) else - tt = nil - break + tt[#tt+1] = tostring(v) end + elseif tv == "boolean" then + tt[#tt+1] = tostring(v) + elseif tv == "string" then + tt[#tt+1] = ("%q"):format(v) + else + tt = nil + break end - return tt end + return tt end - return nil end + return nil +end - local function serialize(root,name,handle,depth,level,reduce,noquotes,indexed) - handle = handle or print - reduce = reduce or false - if depth then - depth = depth .. " " - if indexed then - handle(("%s{"):format(depth)) - else - handle(("%s%s={"):format(depth,key(name))) - end - else - depth = "" - local tname = type(name) - if tname == "string" then - if name == "return" then - handle("return {") - else - handle(name .. "={") - end - elseif tname == "number" then - handle("[" .. name .. "]={") - elseif tname == "boolean" then - if name then - handle("return {") - else - handle("{") - end - else - handle("t={") - end - end - if root and next(root) then - local compact = table.serialize_compact - local inline = compact and table.serialize_inline - local first, last = nil, 0 -- #root cannot be trusted here - if compact then - for k,v in ipairs(root) do -- NOT: for k=1,#root do (why) - if not first then first = k end - last = last + 1 - end - end - for _,k in pairs(table.sortedkeys(root)) do - local v = root[k] - local t = type(v) - if compact and first and type(k) == "number" and k >= first and k <= last then - if t == "number" then - handle(("%s %s,"):format(depth,v)) - elseif t == "string" then - if reduce and (v:find("^[%-%+]?[%d]-%.?[%d+]$") == 1) then - handle(("%s %s,"):format(depth,v)) - else - handle(("%s %q,"):format(depth,v)) - end - elseif t == "table" then - if not next(v) then - handle(("%s {},"):format(depth)) - elseif inline then - local st = simple_table(v) - if st then - handle(("%s { %s },"):format(depth,table.concat(st,", "))) - else - serialize(v,k,handle,depth,level+1,reduce,noquotes,true) - end - else - serialize(v,k,handle,depth,level+1,reduce,noquotes,true) - end - elseif t == "boolean" then - handle(("%s %s,"):format(depth,tostring(v))) - elseif t == "function" then - if table.serialize_functions then - handle(('%s loadstring(%q),'):format(depth,string.dump(v))) - else - handle(('%s "function",'):format(depth)) - end +local function do_serialize(root,name,depth,level,indexed) + if level > 0 then + depth = depth .. " " + if indexed then + handle(("%s{"):format(depth)) + elseif name then + handle(("%s%s={"):format(depth,key(name))) + else + handle(("%s{"):format(depth)) + end + end + if root and next(root) then + local first, last = nil, 0 -- #root cannot be trusted here + if compact then + for k,v in ipairs(root) do -- NOT: for k=1,#root do (we need to quit at nil) + if not first then first = k end + last = last + 1 + end + end + --~ for _,k in pairs(sortedkeys(root)) do -- 1% faster: + local sk = sortedkeys(root) + for i=1,#sk do + local k = sk[i] + local v = root[k] + local t = type(v) + if compact and first and type(k) == "number" and k >= first and k <= last then + if t == "number" then + if hexify then + handle(("%s 0x%04X,"):format(depth,v)) else - handle(("%s %q,"):format(depth,tostring(v))) - end - elseif k == "__p__" then -- parent - if false then - handle(("%s __p__=nil,"):format(depth)) + handle(("%s %s,"):format(depth,v)) end - elseif t == "number" then - handle(("%s %s=%s,"):format(depth,key(k),v)) elseif t == "string" then if reduce and (v:find("^[%-%+]?[%d]-%.?[%d+]$") == 1) then - handle(("%s %s=%s,"):format(depth,key(k),v)) + handle(("%s %s,"):format(depth,v)) else - handle(("%s %s=%q,"):format(depth,key(k),v)) + handle(("%s %q,"):format(depth,v)) end elseif t == "table" then if not next(v) then - handle(("%s %s={},"):format(depth,key(k))) + handle(("%s {},"):format(depth)) elseif inline then local st = simple_table(v) if st then - handle(("%s %s={ %s },"):format(depth,key(k),table.concat(st,", "))) + handle(("%s { %s },"):format(depth,concat(st,", "))) else - serialize(v,k,handle,depth,level+1,reduce,noquotes) + do_serialize(v,k,depth,level+1,true) end else - serialize(v,k,handle,depth,level+1,reduce,noquotes) + do_serialize(v,k,depth,level+1,true) end elseif t == "boolean" then - handle(("%s %s=%s,"):format(depth,key(k),tostring(v))) + handle(("%s %s,"):format(depth,tostring(v))) elseif t == "function" then - if table.serialize_functions then - handle(('%s %s=loadstring(%q),'):format(depth,key(k),string.dump(v))) + if functions then + handle(('%s loadstring(%q),'):format(depth,v:dump())) else - handle(('%s %s="function",'):format(depth,key(k))) + handle(('%s "function",'):format(depth)) end else - handle(("%s %s=%q,"):format(depth,key(k),tostring(v))) - -- handle(('%s %s=loadstring(%q),'):format(depth,key(k),string.dump(function() return v end))) + handle(("%s %q,"):format(depth,tostring(v))) + end + elseif k == "__p__" then -- parent + if false then + handle(("%s __p__=nil,"):format(depth)) + end + elseif t == "number" then + if hexify then + handle(("%s %s=0x%04X,"):format(depth,key(k),v)) + else + handle(("%s %s=%s,"):format(depth,key(k),v)) + end + elseif t == "string" then + if reduce and (v:find("^[%-%+]?[%d]-%.?[%d+]$") == 1) then + handle(("%s %s=%s,"):format(depth,key(k),v)) + else + handle(("%s %s=%q,"):format(depth,key(k),v)) + end + elseif t == "table" then + if not next(v) then + handle(("%s %s={},"):format(depth,key(k))) + elseif inline then + local st = simple_table(v) + if st then + handle(("%s %s={ %s },"):format(depth,key(k),concat(st,", "))) + else + do_serialize(v,k,depth,level+1) + end + else + do_serialize(v,k,depth,level+1) + end + elseif t == "boolean" then + handle(("%s %s=%s,"):format(depth,key(k),tostring(v))) + elseif t == "function" then + if functions then + handle(('%s %s=loadstring(%q),'):format(depth,key(k),v:dump())) + else + handle(('%s %s="function",'):format(depth,key(k))) end - end - if level > 0 then - handle(("%s},"):format(depth)) else - handle(("%s}"):format(depth)) + handle(("%s %s=%q,"):format(depth,key(k),tostring(v))) + -- handle(('%s %s=loadstring(%q),'):format(depth,key(k),string.dump(function() return v end))) end - else - handle(("%s}"):format(depth)) end end + if level > 0 then + handle(("%s},"):format(depth)) + end +end - --~ name: - --~ - --~ true : return { } - --~ false : { } - --~ nil : t = { } - --~ string : string = { } - --~ 'return' : return { } - --~ number : [number] = { } - - function table.serialize(root,name,reduce,noquotes) - local t = { } - local function flush(s) - t[#t+1] = s +local function serialize(root,name,_handle,_reduce,_noquotes,_hexify) + noquotes = _noquotes + hexify = _hexify + handle = _handle or print + reduce = _reduce or false + compact = table.serialize_compact + inline = compact and table.serialize_inline + functions = table.serialize_functions + local tname = type(name) + if tname == "string" then + if name == "return" then + handle("return {") + else + handle(name .. "={") + end + elseif tname == "number" then + if hexify then + handle(("[0x%04X]={"):format(name)) + else + handle("[" .. name .. "]={") + end + elseif tname == "boolean" then + if name then + handle("return {") + else + handle("{") end - serialize(root, name, flush, nil, 0, reduce, noquotes) - return table.concat(t,"\n") + else + handle("t={") + end + if root and next(root) then + do_serialize(root,name,"",0,indexed) end + handle("}") +end + +--~ name: +--~ +--~ true : return { } +--~ false : { } +--~ nil : t = { } +--~ string : string = { } +--~ 'return' : return { } +--~ number : [number] = { } - function table.tohandle(handle,root,name,reduce,noquotes) - serialize(root, name, handle, nil, 0, reduce, noquotes) +function table.serialize(root,name,reduce,noquotes,hexify) + local t = { } + local function flush(s) + t[#t+1] = s end + serialize(root,name,flush,reduce,noquotes,hexify) + return concat(t,"\n") +end - -- sometimes tables are real use (zapfino extra pro is some 85M) in which - -- case a stepwise serialization is nice; actually, we could consider: - -- - -- for line in table.serializer(root,name,reduce,noquotes) do - -- ...(line) - -- end - -- - -- so this is on the todo list +function table.tohandle(handle,root,name,reduce,noquotes,hexify) + serialize(root,name,handle,reduce,noquotes,hexify) +end - table.tofile_maxtab = 2*1024 +-- sometimes tables are real use (zapfino extra pro is some 85M) in which +-- case a stepwise serialization is nice; actually, we could consider: +-- +-- for line in table.serializer(root,name,reduce,noquotes) do +-- ...(line) +-- end +-- +-- so this is on the todo list - function table.tofile(filename,root,name,reduce,noquotes) - local f = io.open(filename,'w') - if f then - local concat = table.concat - local maxtab = table.tofile_maxtab - if maxtab > 1 then - local t = { } - local function flush(s) - t[#t+1] = s - if #t > maxtab then - f:write(concat(t,"\n"),"\n") -- hm, write(sometable) should be nice - t = { } - end - end - serialize(root, name, flush, nil, 0, reduce, noquotes) - f:write(concat(t,"\n"),"\n") - else - local function flush(s) - f:write(s,"\n") +table.tofile_maxtab = 2*1024 + +function table.tofile(filename,root,name,reduce,noquotes,hexify) + local f = io.open(filename,'w') + if f then + local maxtab = table.tofile_maxtab + if maxtab > 1 then + local t = { } + local function flush(s) + t[#t+1] = s + if #t > maxtab then + f:write(concat(t,"\n"),"\n") -- hm, write(sometable) should be nice + t = { } end - serialize(root, name, flush, nil, 0, reduce, noquotes) end - f:close() + serialize(root,name,flush,reduce,noquotes,hexify) + f:write(concat(t,"\n"),"\n") + else + local function flush(s) + f:write(s,"\n") + end + serialize(root,name,flush,reduce,noquotes,hexify) end + f:close() end - end ---~ t = { ---~ b = "123", ---~ a = "x", ---~ c = 1.23, ---~ d = "1.23", ---~ e = true, ---~ f = { ---~ d = "1.23", ---~ a = "x", ---~ b = "123", ---~ c = 1.23, ---~ e = true, ---~ f = { ---~ e = true, ---~ f = { ---~ e = true ---~ }, ---~ }, ---~ }, ---~ g = function() end ---~ } - ---~ print(table.serialize(t), "\n") ---~ print(table.serialize(t,"name"), "\n") ---~ print(table.serialize(t,false), "\n") ---~ print(table.serialize(t,true), "\n") ---~ print(table.serialize(t,"name",true), "\n") ---~ print(table.serialize(t,"name",true,true), "\n") - -do - - local function flatten(t,f,complete) - for i=1,#t do - local v = t[i] - if type(v) == "table" then - if complete or type(v[1]) == "table" then - flatten(v,f,complete) - else - f[#f+1] = v - end +local function flatten(t,f,complete) + for i=1,#t do + local v = t[i] + if type(v) == "table" then + if complete or type(v[1]) == "table" then + flatten(v,f,complete) else f[#f+1] = v end + else + f[#f+1] = v end end +end - function table.flatten(t) - local f = { } - flatten(t,f,true) - return f - end +function table.flatten(t) + local f = { } + flatten(t,f,true) + return f +end - function table.unnest(t) -- bad name - local f = { } - flatten(t,f,false) - return f - end +function table.unnest(t) -- bad name + local f = { } + flatten(t,f,false) + return f +end + +table.flatten_one_level = table.unnest - table.flatten_one_level = table.unnest +-- the next three may disappear +function table.remove_value(t,value) -- todo: n + if value then + for i=1,#t do + if t[i] == value then + remove(t,i) + -- remove all, so no: return + end + end + end end function table.insert_before_value(t,value,str) - for i=1,#t do - if t[i] == value then - table.insert(t,i,str) - return + if str then + if value then + for i=1,#t do + if t[i] == value then + insert(t,i,str) + return + end + end end + insert(t,1,str) + elseif value then + insert(t,1,value) end - table.insert(t,1,str) end function table.insert_after_value(t,value,str) - for i=1,#t do - if t[i] == value then - table.insert(t,i+1,str) - return + if str then + if value then + for i=1,#t do + if t[i] == value then + insert(t,i+1,str) + return + end + end end + t[#t+1] = str + elseif value then + t[#t+1] = value end - t[#t+1] = str end function table.are_equal(a,b,n,m) @@ -1012,27 +1047,11 @@ function table.compact(t) end end -function table.tohash(t) - local h = { } - for _, v in pairs(t) do -- no ipairs here - h[v] = true - end - return h -end - -function table.fromhash(t) - local h = { } - for k, v in pairs(t) do -- no ipairs here - if v then h[#h+1] = k end - end - return h -end - function table.contains(t, v) if t then for i=1, #t do if t[i] == v then - return true + return i end end end @@ -1069,11 +1088,10 @@ function table.clone(t,p) -- t is option return t end - function table.hexed(t,seperator) local tt = { } - for i=1,#t do tt[i] = string.format("0x%04X",t[i]) end - return table.concat(tt,seperator or " ") + for i=1,#t do tt[i] = ("0x%04X"):format(t[i]) end + return concat(tt,seperator or " ") end function table.reverse_hash(h) @@ -1113,6 +1131,7 @@ function io.loaddata(filename) local f = io.open(filename,'rb') if f then local data = f:read('*all') + -- garbagecollector.check(data) f:close() return data else @@ -1121,7 +1140,7 @@ function io.loaddata(filename) end function io.savedata(filename,data,joiner) - local f = io.open(filename, "wb") + local f = io.open(filename,"wb") if f then if type(data) == "table" then f:write(table.join(data,joiner or "")) @@ -1131,6 +1150,9 @@ function io.savedata(filename,data,joine f:write(data) end f:close() + return true + else + return false end end @@ -1465,6 +1487,9 @@ end -- copyright: PRAGMA ADE / ConTeXt Development Team -- license : see context related readme files + +--~ print(table.serialize(os.uname())) + if not versions then versions = { } end versions['l-os'] = 1.001 function os.resultof(command) @@ -1560,10 +1585,14 @@ if not versions then versions = { } end if not file then file = { } end +local concat = table.concat + function file.removesuffix(filename) - return filename:gsub("%.[%a%d]+$", "") + return (filename:gsub("%.[%a%d]+$","")) end +file.stripsuffix = file.removesuffix + function file.addsuffix(filename, suffix) if not filename:find("%.[%a%d]+$") then return filename .. "." .. suffix @@ -1573,11 +1602,7 @@ function file.addsuffix(filename, suffix end function file.replacesuffix(filename, suffix) - if not filename:find("%.[%a%d]+$") then - return filename .. "." .. suffix - else - return (filename:gsub("%.[%a%d]+$","."..suffix)) - end + return (filename:gsub("%.[%a%d]+$","")) .. "." .. suffix end function file.dirname(name) @@ -1598,18 +1623,6 @@ end file.suffix = file.extname -function file.stripsuffix(name) - return (name:gsub("%.[%a%d]+$","")) -end - ---~ function file.join(...) ---~ local t = { ... } ---~ for i=1,#t do ---~ t[i] = (t[i]:gsub("\\","/")):gsub("/+$","") ---~ end ---~ return table.concat(t,"/") ---~ end - --~ print(file.join("x/","/y")) --~ print(file.join("http://","/y")) --~ print(file.join("http://a","/y")) @@ -1617,7 +1630,7 @@ end --~ print(file.join("//nas-1","/y")) function file.join(...) - local pth = table.concat({...},"/") + local pth = concat({...},"/") pth = pth:gsub("\\","/") local a, b = pth:match("^(.*://)(.*)$") if a and b then @@ -1650,6 +1663,16 @@ function file.is_readable(name) end end +function file.iswritable(name) + local a = lfs.attributes(name) + return a and a.permissions:sub(2,2) == "w" +end + +function file.isreadable(name) + local a = lfs.attributes(name) + return a and a.permissions:sub(1,1) == "r" +end + --~ function file.split_path(str) --~ if str:find(';') then --~ return str:splitchr(";") @@ -1674,37 +1697,29 @@ function file.split_path(str) end function file.join_path(tab) - return table.concat(tab,io.pathseparator) -- can have trailing // + return concat(tab,io.pathseparator) -- can have trailing // end ---~ print('test' .. " == " .. file.collapse_path("test")) ---~ print("test/test" .. " == " .. file.collapse_path("test/test")) ---~ print("test/test/test" .. " == " .. file.collapse_path("test/test/test")) ---~ print("test/test" .. " == " .. file.collapse_path("test/../test/test")) ---~ print("test" .. " == " .. file.collapse_path("test/../test")) ---~ print("../test" .. " == " .. file.collapse_path("../test")) ---~ print("../test/" .. " == " .. file.collapse_path("../test/")) ---~ print("a/a" .. " == " .. file.collapse_path("a/b/c/../../a")) - ---~ function file.collapse_path(str) ---~ local ok, n = false, 0 ---~ while not ok do ---~ ok = true ---~ str, n = str:gsub("[^%./]+/%.%./", function(s) ---~ ok = false ---~ return "" ---~ end) ---~ end ---~ return (str:gsub("/%./","/")) ---~ end - function file.collapse_path(str) - local n = 1 - while n > 0 do - str, n = str:gsub("([^/%.]+/%.%./)","") - end - return (str:gsub("/%./","/")) -end + str = str:gsub("/%./","/") + local n, m = 1, 1 + while n > 0 or m > 0 do + str, n = str:gsub("[^/%.]+/%.%.$","") + str, m = str:gsub("[^/%.]+/%.%./","") + end + str = str:gsub("([^/])/$","%1") + str = str:gsub("^%./","") + str = str:gsub("/%.$","") + if str == "" then str = "." end + return str +end + +--~ print(file.collapse_path("a/./b/..")) +--~ print(file.collapse_path("a/aa/../b/bb")) +--~ print(file.collapse_path("a/../..")) +--~ print(file.collapse_path("a/.././././b/..")) +--~ print(file.collapse_path("a/./././b/..")) +--~ print(file.collapse_path("a/b/c/../..")) function file.robustname(str) return (str:gsub("[^%a%d%/%-%.\\]+","-")) @@ -1717,19 +1732,111 @@ function file.copy(oldname,newname) file.savedata(newname,io.loaddata(oldname)) end +-- lpeg variants, slightly faster, not always --- filename : l-url.lua --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +--~ local period = lpeg.P(".") +--~ local slashes = lpeg.S("\\/") +--~ local noperiod = 1-period +--~ local noslashes = 1-slashes +--~ local name = noperiod^1 -if not versions then versions = { } end versions['l-url'] = 1.001 -if not url then url = { } end +--~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.C(noperiod^1) * -1 --- from the spec (on the web): --- --- foo://example.com:8042/over/there?name=ferret#nose --- \_/ \______________/\_________/ \_________/ \__/ +--~ function file.extname(name) +--~ return pattern:match(name) or "" +--~ end + +--~ local pattern = lpeg.Cs(((period * noperiod^1 * -1)/"" + 1)^1) + +--~ function file.removesuffix(name) +--~ return pattern:match(name) +--~ end + +--~ file.stripsuffix = file.removesuffix + +--~ local pattern = (noslashes^0 * slashes)^1 * lpeg.C(noslashes^1) * -1 + +--~ function file.basename(name) +--~ return pattern:match(name) or name +--~ end + +--~ local pattern = (noslashes^0 * slashes)^1 * lpeg.Cp() * noslashes^1 * -1 + +--~ function file.dirname(name) +--~ local p = pattern:match(name) +--~ if p then +--~ return name:sub(1,p-2) +--~ else +--~ return "" +--~ end +--~ end + +--~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1 + +--~ function file.addsuffix(name, suffix) +--~ local p = pattern:match(name) +--~ if p then +--~ return name +--~ else +--~ return name .. "." .. suffix +--~ end +--~ end + +--~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1 + +--~ function file.replacesuffix(name,suffix) +--~ local p = pattern:match(name) +--~ if p then +--~ return name:sub(1,p-2) .. "." .. suffix +--~ else +--~ return name .. "." .. suffix +--~ end +--~ end + +--~ local pattern = (noslashes^0 * slashes)^0 * lpeg.Cp() * ((noperiod^1 * period)^1 * lpeg.Cp() + lpeg.P(true)) * noperiod^1 * -1 + +--~ function file.nameonly(name) +--~ local a, b = pattern:match(name) +--~ if b then +--~ return name:sub(a,b-2) +--~ elseif a then +--~ return name:sub(a) +--~ else +--~ return name +--~ end +--~ end + +--~ local test = file.extname +--~ local test = file.stripsuffix +--~ local test = file.basename +--~ local test = file.dirname +--~ local test = file.addsuffix +--~ local test = file.replacesuffix +--~ local test = file.nameonly + +--~ print(1,test("./a/b/c/abd.def.xxx","!!!")) +--~ print(2,test("./../b/c/abd.def.xxx","!!!")) +--~ print(3,test("a/b/c/abd.def.xxx","!!!")) +--~ print(4,test("a/b/c/def.xxx","!!!")) +--~ print(5,test("a/b/c/def","!!!")) +--~ print(6,test("def","!!!")) +--~ print(7,test("def.xxx","!!!")) + +--~ local tim = os.clock() for i=1,250000 do local ext = test("abd.def.xxx","!!!") end print(os.clock()-tim) + + +-- filename : l-url.lua +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-url'] = 1.001 +if not url then url = { } end + +-- from the spec (on the web): +-- +-- foo://example.com:8042/over/there?name=ferret#nose +-- \_/ \______________/\_________/ \_________/ \__/ -- | | | | | -- scheme authority path query fragment -- | _____________________|__ @@ -1837,51 +1944,6 @@ dir = { } if lfs then do ---~ local attributes = lfs.attributes ---~ local walkdir = lfs.dir ---~ ---~ local function glob_pattern(path,patt,recurse,action) ---~ local ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe ---~ if ok and type(scanner) == "function" then ---~ if not path:find("/$") then path = path .. '/' end ---~ for name in scanner do ---~ local full = path .. name ---~ local mode = attributes(full,'mode') ---~ if mode == 'file' then ---~ if name:find(patt) then ---~ action(full) ---~ end ---~ elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then ---~ glob_pattern(full,patt,recurse,action) ---~ end ---~ end ---~ end ---~ end ---~ ---~ dir.glob_pattern = glob_pattern ---~ ---~ local function glob(pattern, action) ---~ local t = { } ---~ local action = action or function(name) t[#t+1] = name end ---~ local path, patt = pattern:match("^(.*)/*%*%*/*(.-)$") ---~ local recurse = path and patt ---~ if not recurse then ---~ path, patt = pattern:match("^(.*)/(.-)$") ---~ if not (path and patt) then ---~ path, patt = '.', pattern ---~ end ---~ end ---~ patt = patt:gsub("([%.%-%+])", "%%%1") ---~ patt = patt:gsub("%*", ".*") ---~ patt = patt:gsub("%?", ".") ---~ patt = "^" .. patt .. "$" ---~ -- print('path: ' .. path .. ' | pattern: ' .. patt .. ' | recurse: ' .. tostring(recurse)) ---~ glob_pattern(path,patt,recurse,action) ---~ return t ---~ end ---~ ---~ dir.glob = glob - local attributes = lfs.attributes local walkdir = lfs.dir @@ -1910,30 +1972,6 @@ if lfs then do dir.glob_pattern = glob_pattern - --~ local function glob(pattern, action) - --~ local t = { } - --~ local path, rest, patt, recurse - --~ local action = action or function(name) t[#t+1] = name end - --~ local pattern = pattern:gsub("^%*%*","./**") - --~ local pattern = pattern:gsub("/%*/","/**/") - --~ path, rest = pattern:match("^(/)(.-)$") - --~ if path then - --~ path = path - --~ else - --~ path, rest = pattern:match("^([^/]*)/(.-)$") - --~ end - --~ if rest then - --~ patt = rest:gsub("([%.%-%+])", "%%%1") - --~ end - --~ patt = patt:gsub("%*", "[^/]*") - --~ patt = patt:gsub("%?", "[^/]") - --~ patt = patt:gsub("%[%^/%]%*%[%^/%]%*", ".*") - --~ if path == "" then path = "." end - --~ recurse = patt:find("%.%*/") ~= nil - --~ glob_pattern(path,patt,recurse,action) - --~ return t - --~ end - local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V local pattern = Ct { @@ -1959,13 +1997,17 @@ if lfs then do glob(s,t) end return t + elseif lfs.isfile(str) then + local t = t or { } + t[#t+1] = str + return t else local split = pattern:match(str) if split then local t = t or { } local action = action or function(name) t[#t+1] = name end local root, path, base = split[1], split[2], split[3] - local recurse = base:find("**") + local recurse = base:find("%*%*") local start = root .. path local result = filter:match(start .. base) glob_pattern(start,result,recurse,action) @@ -1993,16 +2035,21 @@ if lfs then do for name in walkdir(path) do if name:find("^%.") then --- skip - elseif attributes(name,'mode') == "directory" then - if recurse then - globfiles(path .. "/" .. name,recurse,func,files) - end - elseif func then - if func(name) then - files[#files+1] = path .. "/" .. name - end else - files[#files+1] = path .. "/" .. name + local mode = attributes(name,'mode') + if mode == "directory" then + if recurse then + globfiles(path .. "/" .. name,recurse,func,files) + end + elseif mode == "file" then + if func then + if func(name) then + files[#files+1] = path .. "/" .. name + end + else + files[#files+1] = path .. "/" .. name + end + end end end return files @@ -2210,7 +2257,7 @@ function toboolean(str,tolerant) if tolerant then local tstr = type(str) if tstr == "string" then - return str == "true" or str == "yes" or str == "on" or str == "1" + return str == "true" or str == "yes" or str == "on" or str == "1" or str == "t" elseif tstr == "number" then return tonumber(str) ~= 0 elseif tstr == "nil" then @@ -2229,9 +2276,9 @@ end function string.is_boolean(str) if type(str) == "string" then - if str == "true" or str == "yes" or str == "on" then + if str == "true" or str == "yes" or str == "on" or str == "t" then return true - elseif str == "false" or str == "no" or str == "off" then + elseif str == "false" or str == "no" or str == "off" or str == "f" then return false end end @@ -2256,6 +2303,9 @@ end if not versions then versions = { } end versions['l-unicode'] = 1.001 if not unicode then unicode = { } end +local concat, utfchar, utfgsub = table.concat, unicode.utf8.char, unicode.utf8.gsub +local char, byte = string.char, string.byte + if not garbagecollector then garbagecollector = { push = function() collectgarbage("stop") end, @@ -2301,70 +2351,69 @@ function unicode.utftype(f) -- \000 fail end end -function unicode.utf16_to_utf8(str, endian) - garbagecollector.push() - local result = { } - local tc, uc = table.concat, unicode.utf8.char - local tmp, n, m, p = { }, 0, 0, 0 +function unicode.utf16_to_utf8(str, endian) -- maybe a gsub is faster or an lpeg +--~ garbagecollector.push() + local result, tmp, n, m, p = { }, { }, 0, 0, 0 -- lf | cr | crlf / (cr:13, lf:10) local function doit() if n == 10 then if p ~= 13 then - result[#result+1] = tc(tmp,"") + result[#result+1] = concat(tmp,"") tmp = { } p = 0 end elseif n == 13 then - result[#result+1] = tc(tmp,"") + result[#result+1] = concat(tmp,"") tmp = { } p = n else - tmp[#tmp+1] = uc(n) + tmp[#tmp+1] = utfchar(n) p = 0 end end for l,r in str:bytepairs() do - if endian then - n = l*256 + r - else - n = r*256 + l - end - if m > 0 then - n = (m-0xD800)*0x400 + (n-0xDC00) + 0x10000 - m = 0 - doit() - elseif n >= 0xD800 and n <= 0xDBFF then - m = n - else - doit() + if r then + if endian then + n = l*256 + r + else + n = r*256 + l + end + if m > 0 then + n = (m-0xD800)*0x400 + (n-0xDC00) + 0x10000 + m = 0 + doit() + elseif n >= 0xD800 and n <= 0xDBFF then + m = n + else + doit() + end end end if #tmp > 0 then - result[#result+1] = tc(tmp,"") + result[#result+1] = concat(tmp,"") end - garbagecollector.pop() +--~ garbagecollector.pop() return result end function unicode.utf32_to_utf8(str, endian) - garbagecollector.push() +--~ garbagecollector.push() local result = { } - local tc, uc = table.concat, unicode.utf8.char local tmp, n, m, p = { }, 0, -1, 0 -- lf | cr | crlf / (cr:13, lf:10) local function doit() if n == 10 then if p ~= 13 then - result[#result+1] = tc(tmp,"") + result[#result+1] = concat(tmp,"") tmp = { } p = 0 end elseif n == 13 then - result[#result+1] = tc(tmp,"") + result[#result+1] = concat(tmp,"") tmp = { } p = n else - tmp[#tmp+1] = uc(n) + tmp[#tmp+1] = utfchar(n) p = 0 end end @@ -2390,12 +2439,67 @@ function unicode.utf32_to_utf8(str, endi end end if #tmp > 0 then - result[#result+1] = tc(tmp,"") + result[#result+1] = concat(tmp,"") end - garbagecollector.pop() +--~ garbagecollector.pop() return result end +function unicode.utf8_to_utf16(str,littleendian) + if littleendian then + return char(255,254) .. utfgsub(str,".",function(c) + local b = byte(c) + if b < 0x10000 then + return char(b%256,b/256) + else + b = b - 0x10000 + local b1, b2 = b/1024 + 0xD800, b%1024 + 0xDC00 + return char(b1%256,b1/256,b2%256,b2/256) + end + end) + else + return char(254,255) .. utfgsub(str,".",function(c) + local b = byte(c) + if b < 0x10000 then + return char(b/256,b%256) + else + b = b - 0x10000 + local b1, b2 = b/1024 + 0xD800, b%1024 + 0xDC00 + return char(b1/256,b1%256,b2/256,b2%256) + end + end) + end +end + + +-- filename : l-math.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-math'] = 1.001 + +local floor = math.floor + +if not math.round then + function math.round(x) + return floor(x + 0.5) + end +end + +if not math.div then + function math.div(n,m) + return floor(n/m) + end +end + +if not math.mod then + function math.mod(n,m) + return n % m + end +end + -- filename : l-utils.lua -- comment : split off from luat-lib @@ -2429,11 +2533,20 @@ function utils.report(...) print(...) end +utils.merger.strip_comment = true + function utils.merger._self_load_(name) local f, data = io.open(name), "" if f then + utils.report("reading merge from %s",name) data = f:read("*all") f:close() + else + utils.report("unknown file to merge %s",name) + end + if data and utils.merger.strip_comment then + -- saves some 20K + data = data:gsub("%-%-~[^\n\r]*[\r\n]", "") end return data or "" end @@ -2442,6 +2555,7 @@ function utils.merger._self_save_(name, if data ~= "" then local f = io.open(name,'w') if f then + utils.report("saving merge from %s",name) f:write(data) f:close() end @@ -2467,13 +2581,13 @@ function utils.merger._self_libs_(libs,l local name = string.gsub(pth .. "/" .. lib,"\\","/") f = io.open(name) if f then - -- utils.report("merging library",name) + utils.report("merging library %s",name) result[#result+1] = f:read("*all") f:close() list = { pth } -- speed up the search break else - -- utils.report("no library",name) + utils.report("no library %s",name) end end end @@ -2512,108 +2626,62 @@ function utils.merger.selfclean(name) ) end -utils.lua.compile_strip = true - -function utils.lua.compile(luafile, lucfile) +function utils.lua.compile(luafile, lucfile, cleanup, strip) -- defaults: cleanup=false strip=true -- utils.report("compiling",luafile,"into",lucfile) os.remove(lucfile) local command = "-o " .. string.quote(lucfile) .. " " .. string.quote(luafile) - if utils.lua.compile_strip then + if strip ~= false then command = "-s " .. command end - if os.spawn("texluac " .. command) == 0 then - return true - elseif os.spawn("luac " .. command) == 0 then - return true - else - return false + local done = (os.spawn("texluac " .. command) == 0) or (os.spawn("luac " .. command) == 0) + if done and cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then + -- utils.report("removing",luafile) + os.remove(luafile) end + return done end --- filename : luat-lib.lua --- comment : companion to luat-lib.tex --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files - -if not versions then versions = { } end versions['luat-lib'] = 1.001 - --- mostcode moved to the l-*.lua and other luat-*.lua files +if not modules then modules = { } end modules ['luat-lib'] = { + version = 1.001, + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files", + comment = "companion to luat-lib.tex", +} --- os / io +-- most code already moved to the l-*.lua and other luat-*.lua files os.setlocale(nil,nil) -- useless feature and even dangerous in luatex --- os.platform - --- mswin|bccwin|mingw|cygwin windows --- darwin|rhapsody|nextstep macosx --- netbsd|unix unix --- linux linux - -if not io.fileseparator then - if string.find(os.getenv("PATH"),";") then - io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" - else - io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" - end -end - -os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix" - --- arg normalization --- --- for k,v in pairs(arg) do print(k,v) end - --- environment - -if not environment then environment = { } end - -environment.ownbin = environment.ownbin or arg[-2] or arg[-1] or arg[0] or "luatex" - -local ownpath = nil -- we could use a metatable here - -function environment.ownpath() - if not ownpath then - for p in string.gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do - local b = file.join(p,environment.ownbin) - if lfs.isfile(b..".exe") or lfs.isfile(b) then - ownpath = p - break - end - end - if not ownpath then ownpath = '.' end - end - return ownpath +function os.setlocale() + -- no way you can mess with it end if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil end -environment.arguments = { } -environment.files = { } -environment.sorted_argument_keys = nil - -environment.platform = os.platform +environment = environment or { } +environment.arguments = { } +environment.files = { } +environment.sortedflags = nil function environment.initialize_arguments(arg) - environment.arguments = { } - environment.files = { } - environment.sorted_argument_keys = nil + local arguments, files = { }, { } + environment.arguments, environment.files, environment.sortedflags = arguments, files, nil for index, argument in pairs(arg) do if index > 0 then local flag, value = argument:match("^%-+(.+)=(.-)$") if flag then - environment.arguments[flag] = string.unquote(value or "") + arguments[flag] = string.unquote(value or "") else flag = argument:match("^%-+(.+)") if flag then - environment.arguments[flag] = true + arguments[flag] = true else - environment.files[#environment.files+1] = argument + files[#files+1] = argument end end end @@ -2634,19 +2702,21 @@ function environment.setargument(name,va environment.arguments[name] = value end -function environment.argument(name) - if environment.arguments[name] then - return environment.arguments[name] - else - if not environment.sorted_argument_keys then - environment.sorted_argument_keys = { } - for _,v in pairs(table.sortedkeys(environment.arguments)) do - table.insert(environment.sorted_argument_keys, "^" .. v) +function environment.argument(name) -- todo: default (plus typecheck on default) + local arguments, sortedflags = environment.arguments, environment.sortedflags + if arguments[name] then + return arguments[name] + else + if not sortedflags then + sortedflags = { } + for _,v in pairs(table.sortedkeys(arguments)) do + sortedflags[#sortedflags+1] = "^" .. v end + environment.sortedflags = sortedflags end - for _,v in pairs(environment.sorted_argument_keys) do + for _,v in ipairs(sortedflags) do if name:find(v) then - return environment.arguments[v:sub(2,#v)] + return arguments[v:sub(2,#v)] end end end @@ -2667,48 +2737,106 @@ function environment.split_arguments(sep return before, after end -function environment.reconstruct_commandline(arg) +--~ function environment.reconstruct_commandline(arg) +--~ if not arg then arg = environment.original_arguments end +--~ local result = { } +--~ for _,a in ipairs(arg) do -- ipairs 1 .. #n +--~ local kk, vv = a:match("^(%-+.-)=(.+)$") +--~ if kk and vv then +--~ if vv:find(" ") then +--~ vv = vv:unquote() +--~ vv = vv:gsub('"','\\"') +--~ result[#result+1] = kk .. "=" .. vv:quote() +--~ else +--~ a = a:unquote() +--~ a = a:gsub('"','\\"') +--~ result[#result+1] = a +--~ end +--~ elseif a:find(" ") then +--~ a = a:unquote() +--~ a = a:gsub('"','\\"') +--~ result[#result+1] = a:quote() +--~ else +--~ result[#result+1] = a +--~ end +--~ end +--~ return table.join(result," ") +--~ end + +function environment.reconstruct_commandline(arg,noquote) if not arg then arg = environment.original_arguments end - local result = { } - for _,a in ipairs(arg) do -- ipairs 1 .. #n - local kk, vv = a:match("^(%-+.-)=(.+)$") - if kk and vv then - if vv:find(" ") then - result[#result+1] = kk .. "=" .. string.quote(vv) + if noquote and #arg == 1 then + local a = arg[1] + a = input.resolve(a) + a = a:unquote() + return a + elseif #arg == 1 then + local result = { } + for _,a in ipairs(arg) do -- ipairs 1 .. #n + a = input.resolve(a) + a = a:unquote() + a = a:gsub('"','\\"') -- tricky + if a:find(" ") then + result[#result+1] = a:quote() else result[#result+1] = a end - elseif a:find(" ") then - result[#result+1] = string.quote(a) - else - result[#result+1] = a end + return table.join(result," ") end - return table.join(result," ") end if arg then - environment.initialize_arguments(arg) - environment.original_arguments = arg + + -- new, reconstruct quoted snippets (maybe better just remnove the " then and add them later) + local newarg, instring = { }, false + + for index, argument in ipairs(arg) do + if argument:find("^\"") then + newarg[#newarg+1] = argument:gsub("^\"","") + if not argument:find("\"$") then + instring = true + end + elseif argument:find("\"$") then + newarg[#newarg] = newarg[#newarg] .. " " .. argument:gsub("\"$","") + instring = false + elseif instring then + newarg[#newarg] = newarg[#newarg] .. " " .. argument + else + newarg[#newarg+1] = argument + end + end + for i=1,-5,-1 do + newarg[i] = arg[i] + end + + environment.initialize_arguments(newarg) + environment.original_arguments = newarg + environment.raw_arguments = arg + arg = { } -- prevent duplicate handling + end --- filename : luat-inp.lua --- comment : companion to luat-lib.tex --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files - --- This lib is multi-purpose and can be loaded again later on so that --- additional functionality becomes available. We will split this --- module in components when we're done with prototyping. +if not modules then modules = { } end modules ['luat-inp'] = { + version = 1.001, + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files", + comment = "companion to luat-lib.tex", +} -- TODO: os.getenv -> os.env[] -- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller) -- TODO: check escaping in find etc, too much, too slow --- This is the first code I wrote for LuaTeX, so it needs some cleanup. +-- This lib is multi-purpose and can be loaded again later on so that +-- additional functionality becomes available. We will split this +-- module in components once we're done with prototyping. This is the +-- first code I wrote for LuaTeX, so it needs some cleanup. Before changing +-- something in this module one can best check with Taco or Hans first; there +-- is some nasty trickery going on that relates to traditional kpse support. -- To be considered: hash key lowercase, first entry in table filename -- (any case), rest paths (so no need for optimization). Or maybe a @@ -2718,12 +2846,6 @@ end -- Beware, loading and saving is overloaded in luat-tmp! -if not versions then versions = { } end versions['luat-inp'] = 1.001 -if not environment then environment = { } end -if not file then file = { } end - -if environment.aleph_mode == nil then environment.aleph_mode = true end -- temp hack - if not input then input = { } end if not input.suffixes then input.suffixes = { } end if not input.formats then input.formats = { } end @@ -2736,7 +2858,7 @@ if not input.hashers then input.hashe if not input.generators then input.generators = { } end -- generate databases if not input.filters then input.filters = { } end -- conversion filters -local format = string.format +local format, concat, sortedkeys = string.format, table.concat, table.sortedkeys input.locators.notfound = { nil } input.hashers.notfound = { nil } @@ -2749,8 +2871,16 @@ input.debug = false input.cnfname = 'texmf.cnf' input.luaname = 'texmfcnf.lua' input.lsrname = 'ls-R' -input.luasuffix = '.tma' -input.lucsuffix = '.tmc' +input.homedir = os.env[os.platform == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~' + +--~ input.luasuffix = 'tma' +--~ input.lucsuffix = 'tmc' + +-- for the moment we have .local but this will disappear +input.cnfdefault = '{$SELFAUTOLOC,$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}' + +-- chances are low that the cnf file is in the bin path +input.cnfdefault = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}' -- we use a cleaned up list / format=any is a wildcard, as is *name @@ -2786,7 +2916,8 @@ input.suffixes['lua'] = { 'lua', 'luc', -- FONTFEATURES = .;$TEXMF/fonts/fea// -- FONTCIDMAPS = .;$TEXMF/fonts/cid// -function input.checkconfigdata(instance) -- not yet ok, no time for debugging now +function input.checkconfigdata() -- not yet ok, no time for debugging now + local instance = input.instance local function fix(varname,default) local proname = varname .. "." .. instance.progname or "crap" local p = instance.environment[proname] @@ -2795,7 +2926,15 @@ function input.checkconfigdata(instance) instance.variables[varname] = default -- or environment? end end - fix("LUAINPUTS" , ".;$TEXINPUTS;$TEXMFSCRIPTS") + local name = os.name + if name == "windows" then + fix("OSFONTDIR", "c:/windows/fonts//") + elseif name == "macosx" then + fix("OSFONTDIR", "$HOME/Library/Fonts//;/Library/Fonts//;/System/Library/Fonts//") + else + -- bad luck + end + fix("LUAINPUTS" , ".;$TEXINPUTS;$TEXMFSCRIPTS") -- no progname, hm fix("FONTFEATURES", ".;$TEXMF/fonts/fea//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") fix("FONTCIDMAPS" , ".;$TEXMF/fonts/cid//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") end @@ -2822,14 +2961,20 @@ input.formats ['sfd'] input.suffixes ['sfd'] = { 'sfd' } input.alternatives['subfont definition files'] = 'sfd' -function input.reset() +-- In practice we will work within one tds tree, but i want to keep +-- the option open to build tools that look at multiple trees, which is +-- why we keep the tree specific data in a table. We used to pass the +-- instance but for practical pusposes we now avoid this and use a +-- instance variable. + +function input.newinstance() local instance = { } instance.rootpath = '' instance.treepath = '' - instance.progname = environment.progname or 'context' - instance.engine = environment.engine or 'luatex' + instance.progname = 'context' + instance.engine = 'luatex' instance.format = '' instance.environment = { } instance.variables = { } @@ -2853,12 +2998,12 @@ function input.reset() instance.cachepath = nil instance.loaderror = false instance.smallcache = false + instance.sortdata = false instance.savelists = true instance.cleanuppaths = true instance.allresults = false instance.pattern = nil -- lists instance.kpseonly = false -- lists - instance.cachefile = 'tmftools' instance.loadtime = 0 instance.starttime = 0 instance.stoptime = 0 @@ -2869,23 +3014,13 @@ function input.reset() instance.fakepaths = { } instance.lsrmode = false - if os.env then - -- store once, freeze and faster - for k,v in pairs(os.env) do - instance.environment[k] = input.bare_variable(v) - end - else - -- we will access os.env frequently - for k,v in pairs({'HOME','TEXMF','TEXMFCNF'}) do - local e = os.getenv(v) - if e then - -- input.report("setting",v,"to",input.bare_variable(e)) - instance.environment[v] = input.bare_variable(e) - end - end + -- store once, freeze and faster (once reset we can best use instance.environment) + + for k,v in pairs(os.env) do + instance.environment[k] = input.bare_variable(v) end - -- cross referencing + -- cross referencing, delayed because we can add suffixes for k, v in pairs(input.suffixes) do for _, vv in pairs(v) do @@ -2899,68 +3034,42 @@ function input.reset() end -function input.reset_hashes(instance) - instance.lists = { } - instance.found = { } -end +input.instance = input.instance or nil -function input.bare_variable(str) -- assumes str is a string - -- return string.gsub(string.gsub(string.gsub(str,"%s+$",""),'^"(.+)"$',"%1"),"^'(.+)'$","%1") - return (str:gsub("\s*([\"\']?)(.+)%1\s*", "%2")) +function input.reset() + input.instance = input.newinstance() + return input.instance end -if texio then - input.log = texio.write_nl -else - input.log = print +function input.reset_hashes() + input.instance.lists = { } + input.instance.found = { } end -function input.simple_logger(kind, name) - if name and name ~= "" then - if input.banner then - input.log(input.banner..kind..": "..name) - else - input.log("<<"..kind..": "..name..">>") - end - else - if input.banner then - input.log(input.banner..kind..": no name") - else - input.log("<<"..kind..": no name>>") - end - end -end - -function input.dummy_logger() +function input.bare_variable(str) -- assumes str is a string + -- return string.gsub(string.gsub(string.gsub(str,"%s+$",""),'^"(.+)"$',"%1"),"^'(.+)'$","%1") + return (str:gsub("\s*([\"\']?)(.+)%1\s*", "%2")) end function input.settrace(n) input.trace = tonumber(n or 0) if input.trace > 0 then - input.logger = input.simple_logger input.verbose = true - else - input.logger = function() end end end -function input.report(...) -- inefficient +input.log = (texio and texio.write_nl) or print + +function input.report(...) if input.verbose then - if input.banner then - input.log(input.banner .. table.concat({...},' ')) - elseif input.logmode() == 'xml' then - input.log("<t>"..table.concat({...},' ').."</t>") - else - input.log("<<"..table.concat({...},' ')..">>") - end + input.log("<<"..format(...)..">>") end end -function input.reportlines(str) - if type(str) == "string" then - str = str:split("\n") +function input.report(...) + if input.trace > 0 then -- extra test + input.log("<<"..format(...)..">>") end - for _,v in pairs(str) do input.report(v) end end input.settrace(tonumber(os.getenv("MTX.INPUT.TRACE") or os.getenv("MTX_INPUT_TRACE") or input.trace or 0)) @@ -2989,7 +3098,7 @@ do instance.stoptime = stoptime instance.loadtime = instance.loadtime + loadtime if report then - input.report('load time', format("%0.3f",loadtime)) + input.report("load time %0.3f",loadtime) end return loadtime end @@ -3005,18 +3114,18 @@ end function input.report_loadtime(instance) if instance then - input.report('total load time', input.elapsedtime(instance)) + input.report('total load time %s', input.elapsedtime(instance)) end end input.loadtime = input.elapsedtime -function input.env(instance,key) - return instance.environment[key] or input.osenv(instance,key) +function input.env(key) + return input.instance.environment[key] or input.osenv(key) end -function input.osenv(instance,key) - local ie = instance.environment +function input.osenv(key) + local ie = input.instance.environment local value = ie[key] if value == nil then -- local e = os.getenv(key) @@ -3034,81 +3143,106 @@ end -- we follow a rather traditional approach: -- -- (1) texmf.cnf given in TEXMFCNF --- (2) texmf.cnf searched in TEXMF/web2c +-- (2) texmf.cnf searched in default variable -- --- for the moment we don't expect a configuration file in a zip +-- also we now follow the stupid route: if not set then just assume *one* +-- cnf file under texmf (i.e. distribution) -function input.identify_cnf(instance) - -- we no longer support treepath and rootpath (was handy for testing); - -- also we now follow the stupid route: if not set then just assume *one* - -- cnf file under texmf (i.e. distribution) - if #instance.cnffiles == 0 then - if input.env(instance,'TEXMFCNF') == "" then - local ownpath = environment.ownpath() or "." - if ownpath then - -- beware, this is tricky on my own system because at that location I do have - -- the raw tree that ends up in the zip; i.e. I cannot test this kind of mess - local function locate(filename,list) - local ownroot = input.normalize_name(file.join(ownpath,"../..")) - if not lfs.isdir(file.join(ownroot,"texmf")) then - ownroot = input.normalize_name(file.join(ownpath,"..")) - if not lfs.isdir(file.join(ownroot,"texmf")) then - input.verbose = true - input.report("error", "unable to identify cnf file") - return +input.ownpath = input.ownpath or nil +input.ownbin = input.ownbin or arg[-2] or arg[-1] or arg[0] or "luatex" +input.autoselfdir = true -- false may be handy for debugging + +function input.getownpath() + if not input.ownpath then + if input.autoselfdir and os.selfdir then + input.ownpath = os.selfdir + else + local binary = input.ownbin + if os.platform == "windows" then + binary = file.replacesuffix(binary,"exe") + end + for p in string.gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do + local b = file.join(p,binary) + if lfs.isfile(b) then + -- we assume that after changing to the path the currentdir function + -- resolves to the real location and use this side effect here; this + -- trick is needed because on the mac installations use symlinks in the + -- path instead of real locations + local olddir = lfs.currentdir() + if lfs.chdir(p) then + local pp = lfs.currentdir() + if input.verbose and p ~= pp then + input.report("following symlink %s to %s",p,pp) end - end - local texmfcnf = file.join(ownroot,"texmf-local/web2c",filename) -- for minimals and myself - if not lfs.isfile(texmfcnf) then - texmfcnf = file.join(ownroot,"texmf/web2c",filename) - if not lfs.isfile(texmfcnf) then - input.verbose = true - input.report("error", "unable to locate",filename) - return + input.ownpath = pp + lfs.chdir(olddir) + else + if input.verbose then + input.report("unable to check path %s",p) end + input.ownpath = p end - table.insert(list,texmfcnf) - local ie = instance.environment - if not ie['SELFAUTOPARENT'] then ie['SELFAUTOPARENT'] = ownroot end - if not ie['TEXMFCNF'] then ie['TEXMFCNF'] = file.dirname(texmfcnf) end - end - locate(input.luaname,instance.luafiles) - locate(input.cnfname,instance.cnffiles) - if #instance.luafiles == 0 and instance.cnffiles == 0 then - input.verbose = true - input.report("error", "unable to locate",filename) - os.exit() + break end - -- here we also assume then TEXMF is set in the distribution, if this trickery is - -- used in the minimals, then users who don't use setuptex are on their own with - -- regards to extra trees - else - input.verbose = true - input.report("error", "unable to identify own path") - os.exit() end - else - local t = input.split_path(input.env(instance,'TEXMFCNF')) - t = input.aux.expanded_path(instance,t) - input.aux.expand_vars(instance,t) - local function locate(filename,list) - for _,v in ipairs(t) do - local texmfcnf = input.normalize_name(file.join(v,filename)) - if lfs.isfile(texmfcnf) then - table.insert(list,texmfcnf) - end + end + if not input.ownpath then input.ownpath = '.' end + end + return input.ownpath +end + +function input.identify_own() + local instance = input.instance + local ownpath = input.getownpath() or lfs.currentdir() + local ie = instance.environment + if ownpath then + if input.env('SELFAUTOLOC') == "" then os.env['SELFAUTOLOC'] = file.collapse_path(ownpath) end + if input.env('SELFAUTODIR') == "" then os.env['SELFAUTODIR'] = file.collapse_path(ownpath .. "/..") end + if input.env('SELFAUTOPARENT') == "" then os.env['SELFAUTOPARENT'] = file.collapse_path(ownpath .. "/../..") end + else + input.verbose = true + input.report("error: unable to locate ownpath") + os.exit() + end + if input.env('TEXMFCNF') == "" then os.env['TEXMFCNF'] = input.cnfdefault end + if input.env('TEXOS') == "" then os.env['TEXOS'] = input.env('SELFAUTODIR') end + if input.env('TEXROOT') == "" then os.env['TEXROOT'] = input.env('SELFAUTOPARENT') end + if input.verbose then + for _,v in ipairs({"SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF"}) do + input.report("variable %s set to %s",v,input.env(v) or "unknown") + end + end + function input.identify_own() end +end + +function input.identify_cnf() + local instance = input.instance + if #instance.cnffiles == 0 then + -- fallback + input.identify_own() + -- the real search + input.expand_variables() + local t = input.split_path(input.env('TEXMFCNF')) + t = input.aux.expanded_path(t) + input.aux.expand_vars(t) -- redundant + local function locate(filename,list) + for _,v in ipairs(t) do + local texmfcnf = input.normalize_name(file.join(v,filename)) + if lfs.isfile(texmfcnf) then + table.insert(list,texmfcnf) end end - locate(input.luaname,instance.luafiles) - locate(input.cnfname,instance.cnffiles) end + locate(input.luaname,instance.luafiles) + locate(input.cnfname,instance.cnffiles) end end -function input.load_cnf(instance) +function input.load_cnf() + local instance = input.instance local function loadoldconfigdata() for _, fname in ipairs(instance.cnffiles) do - input.aux.load_cnf(instance,fname) + input.aux.load_cnf(fname) end end -- instance.cnffiles contain complete names now ! @@ -3123,27 +3257,27 @@ function input.load_cnf(instance) instance.rootpath = file.dirname(instance.rootpath) end instance.rootpath = input.normalize_name(instance.rootpath) - instance.environment['SELFAUTOPARENT'] = instance.rootpath -- just to be sure if instance.lsrmode then loadoldconfigdata() elseif instance.diskcache and not instance.renewcache then - input.loadoldconfig(instance,instance.cnffiles) + input.loadoldconfig(instance.cnffiles) if instance.loaderror then loadoldconfigdata() - input.saveoldconfig(instance) + input.saveoldconfig() end else loadoldconfigdata() if instance.renewcache then - input.saveoldconfig(instance) + input.saveoldconfig() end end - input.aux.collapse_cnf_data(instance) + input.aux.collapse_cnf_data() end - input.checkconfigdata(instance) + input.checkconfigdata() end -function input.load_lua(instance) +function input.load_lua() + local instance = input.instance if #instance.luafiles == 0 then -- yet harmless else @@ -3155,14 +3289,14 @@ function input.load_lua(instance) instance.rootpath = file.dirname(instance.rootpath) end instance.rootpath = input.normalize_name(instance.rootpath) - instance.environment['SELFAUTOPARENT'] = instance.rootpath -- just to be sure - input.loadnewconfig(instance) - input.aux.collapse_cnf_data(instance) + input.loadnewconfig() + input.aux.collapse_cnf_data() end - input.checkconfigdata(instance) + input.checkconfigdata() end -function input.aux.collapse_cnf_data(instance) -- potential optmization: pass start index (setup and configuration are shared) +function input.aux.collapse_cnf_data() -- potential optimization: pass start index (setup and configuration are shared) + local instance = input.instance for _,c in ipairs(instance.order) do for k,v in pairs(c) do if not instance.variables[k] then @@ -3177,21 +3311,22 @@ function input.aux.collapse_cnf_data(ins end end -function input.aux.load_cnf(instance,fname) +function input.aux.load_cnf(fname) + local instance = input.instance fname = input.clean_path(fname) - local lname = fname:gsub("%.%a+$",input.luasuffix) + local lname = file.replacesuffix(fname,'lua') local f = io.open(lname) if f then -- this will go f:close() local dname = file.dirname(fname) if not instance.configuration[dname] then - input.aux.load_configuration(instance,dname,lname) + input.aux.load_configuration(dname,lname) instance.order[#instance.order+1] = instance.configuration[dname] end else f = io.open(fname) if f then - input.report("loading", fname) + input.report("loading %s", fname) local line, data, n, k, v local dname = file.dirname(fname) if not instance.configuration[dname] then @@ -3223,227 +3358,226 @@ function input.aux.load_cnf(instance,fna end f:close() else - input.report("skipping", fname) + input.report("skipping %s", fname) end end end -- database loading -function input.load_hash(instance) - input.locatelists(instance) +function input.load_hash() + local instance = input.instance + input.locatelists() if instance.lsrmode then - input.loadlists(instance) + input.loadlists() elseif instance.diskcache and not instance.renewcache then - input.loadfiles(instance) + input.loadfiles() if instance.loaderror then - input.loadlists(instance) - input.savefiles(instance) + input.loadlists() + input.savefiles() end else - input.loadlists(instance) + input.loadlists() if instance.renewcache then - input.savefiles(instance) + input.savefiles() end end end -function input.aux.append_hash(instance,type,tag,name) - input.logger("= hash append",tag) - table.insert(instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } ) +function input.aux.append_hash(type,tag,name) + if input.trace > 0 then + input.logger("= hash append: %s",tag) + end + table.insert(input.instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } ) end -function input.aux.prepend_hash(instance,type,tag,name) - input.logger("= hash prepend",tag) - table.insert(instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } ) +function input.aux.prepend_hash(type,tag,name) + if input.trace > 0 then + input.logger("= hash prepend: %s",tag) + end + table.insert(input.instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } ) end -function input.aux.extend_texmf_var(instance,specification) -- crap - if instance.environment['TEXMF'] then - input.report("extending environment variable TEXMF with", specification) - instance.environment['TEXMF'] = instance.environment['TEXMF']:gsub("^%{", function() - return "{" .. specification .. "," - end) - elseif instance.variables['TEXMF'] then - input.report("extending configuration variable TEXMF with", specification) - instance.variables['TEXMF'] = instance.variables['TEXMF']:gsub("^%{", function() - return "{" .. specification .. "," - end) +function input.aux.extend_texmf_var(specification) -- crap, we could better prepend the hash + local instance = input.instance +-- local t = input.expanded_path_list('TEXMF') -- full expansion + local t = input.split_path(input.env('TEXMF')) + table.insert(t,1,specification) + local newspec = table.join(t,";") + if instance.environment["TEXMF"] then + instance.environment["TEXMF"] = newspec + elseif instance.variables["TEXMF"] then + instance.variables["TEXMF"] = newspec else - input.report("setting configuration variable TEXMF to", specification) - instance.variables['TEXMF'] = "{" .. specification .. "}" + -- weird end - if instance.variables['TEXMF']:find("%,") and not instance.variables['TEXMF']:find("^%{") then - input.report("adding {} to complex TEXMF variable, best do that yourself") - instance.variables['TEXMF'] = "{" .. instance.variables['TEXMF'] .. "}" - end - input.expand_variables(instance) - input.reset_hashes(instance) + input.expand_variables() + input.reset_hashes() end -- locators -function input.locatelists(instance) - for _, path in pairs(input.simplified_list(input.expansion(instance,'TEXMF'))) do - path = file.collapse_path(path) - input.report("locating list of",path) - input.locatedatabase(instance,input.normalize_name(path)) +function input.locatelists() + local instance = input.instance + for _, path in pairs(input.clean_path_list('TEXMF')) do + input.report("locating list of %s",path) + input.locatedatabase(input.normalize_name(path)) end end -function input.locatedatabase(instance,specification) - return input.methodhandler('locators', instance, specification) +function input.locatedatabase(specification) + return input.methodhandler('locators', specification) end -function input.locators.tex(instance,specification) +function input.locators.tex(specification) if specification and specification ~= '' and lfs.isdir(specification) then - input.logger('! tex locator', specification..' found') - input.aux.append_hash(instance,'file',specification,filename) - else - input.logger('? tex locator', specification..' not found') + if input.trace > 0 then + input.logger('! tex locator found: %s',specification) + end + input.aux.append_hash('file',specification,filename) + elseif input.trace > 0 then + input.logger('? tex locator not found: %s',specification) end end -- hashers -function input.hashdatabase(instance,tag,name) - return input.methodhandler('hashers',instance,tag,name) +function input.hashdatabase(tag,name) + return input.methodhandler('hashers',tag,name) end -function input.loadfiles(instance) +function input.loadfiles() + local instance = input.instance instance.loaderror = false instance.files = { } if not instance.renewcache then for _, hash in ipairs(instance.hashes) do - input.hashdatabase(instance,hash.tag,hash.name) + input.hashdatabase(hash.tag,hash.name) if instance.loaderror then break end end end end -function input.hashers.tex(instance,tag,name) - input.aux.load_files(instance,tag) +function input.hashers.tex(tag,name) + input.aux.load_files(tag) end -- generators: -function input.loadlists(instance) - for _, hash in ipairs(instance.hashes) do - input.generatedatabase(instance,hash.tag) +function input.loadlists() + for _, hash in ipairs(input.instance.hashes) do + input.generatedatabase(hash.tag) end end -function input.generatedatabase(instance,specification) - return input.methodhandler('generators', instance, specification) -end - -do - - local weird = lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) - - function input.generators.tex(instance,specification) - local tag = specification - if not instance.lsrmode and lfs and lfs.dir then - input.report("scanning path",specification) - instance.files[tag] = { } - local files = instance.files[tag] - local n, m, r = 0, 0, 0 - local spec = specification .. '/' - local attributes = lfs.attributes - local directory = lfs.dir - local small = instance.smallcache - local function action(path) - local mode, full - if path then - full = spec .. path .. '/' +function input.generatedatabase(specification) + return input.methodhandler('generators', specification) +end + +local weird = lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) + +function input.generators.tex(specification) + local instance = input.instance + local tag = specification + if not instance.lsrmode and lfs.dir then + input.report("scanning path %s",specification) + instance.files[tag] = { } + local files = instance.files[tag] + local n, m, r = 0, 0, 0 + local spec = specification .. '/' + local attributes = lfs.attributes + local directory = lfs.dir + local small = instance.smallcache + local function action(path) + local mode, full + if path then + full = spec .. path .. '/' + else + full = spec + end + for name in directory(full) do + if name:find("^%.") then + -- skip + -- elseif name:find("[%~%`%!%#%$%%%^%&%*%(%)%=%{%}%[%]%:%;\"\'%|%<%>%,%?\n\r\t]") then -- too much escaped + elseif weird:match(name) then + -- texio.write_nl("skipping " .. name) + -- skip else - full = spec - end - for name in directory(full) do - if name:find("^%.") then - -- skip - -- elseif name:find("[%~%`%!%#%$%%%^%&%*%(%)%=%{%}%[%]%:%;\"\'%|%<%>%,%?\n\r\t]") then -- too much escaped - elseif weird:match(name) then - -- texio.write_nl("skipping " .. name) - -- skip - else - mode = attributes(full..name,'mode') - if mode == "directory" then - m = m + 1 - if path then - action(path..'/'..name) - else - action(name) - end - elseif path and mode == 'file' then - n = n + 1 - local f = files[name] - if f then - if not small then - if type(f) == 'string' then - files[name] = { f, path } - else - f[#f+1] = path - end - end - else - files[name] = path - local lower = name:lower() - if name ~= lower then - files["remap:"..lower] = name - r = r + 1 - end - end + mode = attributes(full..name,'mode') + if mode == 'directory' then + m = m + 1 + if path then + action(path..'/'..name) + else + action(name) end - end - end - end - action() - input.report(format("%s files found on %s directories with %s uppercase remappings",n,m,r)) - else - local fullname = file.join(specification,input.lsrname) - local path = '.' - local f = io.open(fullname) - if f then - instance.files[tag] = { } - local files = instance.files[tag] - local small = instance.smallcache - input.report("loading lsr file",fullname) - -- for line in f:lines() do -- much slower then the next one - for line in (f:read("*a")):gmatch("(.-)\n") do - if line:find("^[%a%d]") then - local fl = files[line] - if fl then + elseif path and mode == 'file' then + n = n + 1 + local f = files[name] + if f then if not small then - if type(fl) == 'string' then - files[line] = { fl, path } -- table + if type(f) == 'string' then + files[name] = { f, path } else - fl[#fl+1] = path + f[#f+1] = path end end else - files[line] = path -- string - local lower = line:lower() - if line ~= lower then - files["remap:"..lower] = line + files[name] = path + local lower = name:lower() + if name ~= lower then + files["remap:"..lower] = name + r = r + 1 + end + end + end + end + end + end + action() + input.report("%s files found on %s directories with %s uppercase remappings",n,m,r) + else + local fullname = file.join(specification,input.lsrname) + local path = '.' + local f = io.open(fullname) + if f then + instance.files[tag] = { } + local files = instance.files[tag] + local small = instance.smallcache + input.report("loading lsr file %s",fullname) + -- for line in f:lines() do -- much slower then the next one + for line in (f:read("*a")):gmatch("(.-)\n") do + if line:find("^[%a%d]") then + local fl = files[line] + if fl then + if not small then + if type(fl) == 'string' then + files[line] = { fl, path } -- table + else + fl[#fl+1] = path end end else - path = line:match("%.%/(.-)%:$") or path -- match could be nil due to empty line + files[line] = path -- string + local lower = line:lower() + if line ~= lower then + files["remap:"..lower] = line + end end + else + path = line:match("%.%/(.-)%:$") or path -- match could be nil due to empty line end - f:close() end + f:close() end end - end -- savers, todo -function input.savefiles(instance) - input.aux.save_data(instance, 'files', function(k,v) - return instance.validfile(k,v) -- path, name +function input.savefiles() + input.aux.save_data('files', function(k,v) + return input.instance.validfile(k,v) -- path, name end) end @@ -3451,8 +3585,8 @@ end -- we join them and split them after the expansion has taken place. This -- is more convenient. -function input.splitconfig(instance) - for i,c in ipairs(instance) do +function input.splitconfig() + for i,c in ipairs(input.instance) do for k,v in pairs(c) do if type(v) == 'string' then local t = file.split_path(v) @@ -3463,8 +3597,9 @@ function input.splitconfig(instance) end end end -function input.joinconfig(instance) - for i,c in ipairs(instance.order) do + +function input.joinconfig() + for i,c in ipairs(input.instance.order) do for k,v in pairs(c) do if type(v) == 'table' then c[k] = file.join_path(v) @@ -3487,8 +3622,9 @@ function input.join_path(str) end end -function input.splitexpansions(instance) - for k,v in pairs(instance.expansions) do +function input.splitexpansions() + local ie = input.instance.expansions + for k,v in pairs(ie) do local t, h = { }, { } for _,vv in pairs(file.split_path(v)) do if vv ~= "" and not h[vv] then @@ -3497,19 +3633,19 @@ function input.splitexpansions(instance) end end if #t > 1 then - instance.expansions[k] = t + ie[k] = t else - instance.expansions[k] = t[1] + ie[k] = t[1] end end end -- end of split/join code -function input.saveoldconfig(instance) - input.splitconfig(instance) - input.aux.save_data(instance, 'configuration', nil) - input.joinconfig(instance) +function input.saveoldconfig() + input.splitconfig() + input.aux.save_data('configuration', nil) + input.joinconfig() end input.configbanner = [[ @@ -3526,8 +3662,6 @@ function input.serialize(files) -- luatools and mtxtools are called frequently. Okay, -- we pay a small price for properly tabbed tables. local t = { } - local concat = table.concat - local sorted = table.sortedkeys local function dump(k,v,m) if type(v) == 'string' then return m .. "['" .. k .. "']='" .. v .. "'," @@ -3538,12 +3672,12 @@ function input.serialize(files) end end t[#t+1] = "return {" - if instance.sortdata then - for _, k in pairs(sorted(files)) do + if input.instance.sortdata then + for _, k in pairs(sortedkeys(files)) do local fk = files[k] if type(fk) == 'table' then t[#t+1] = "\t['" .. k .. "']={" - for _, kk in pairs(sorted(fk)) do + for _, kk in pairs(sortedkeys(fk)) do t[#t+1] = dump(kk,fk[kk],"\t\t") end t[#t+1] = "\t}," @@ -3570,11 +3704,11 @@ end if not texmf then texmf = {} end -- no longer needed, at least not here -function input.aux.save_data(instance, dataname, check, makename) -- untested without cache overload - for cachename, files in pairs(instance[dataname]) do +function input.aux.save_data(dataname, check, makename) -- untested without cache overload + for cachename, files in pairs(input.instance[dataname]) do local name = (makename or file.join)(cachename,dataname) - local luaname, lucname = name .. input.luasuffix, name .. input.lucsuffix - input.report("preparing " .. dataname .. " for", luaname) + local luaname, lucname = name .. ".lua", name .. ".luc" + input.report("preparing %s for %s",dataname,cachename) for k, v in pairs(files) do if not check or check(v,k) then -- path, name if type(v) == "table" and #v == 1 then @@ -3592,38 +3726,38 @@ function input.aux.save_data(instance, d time = os.date("%H:%M:%S"), content = files, } - local f = io.open(luaname,'w') - if f then - input.report("saving " .. dataname .. " in", luaname) - f:write(input.serialize(data)) - f:close() - input.report("compiling " .. dataname .. " to", lucname) - if not utils.lua.compile(luaname,lucname) then - input.report("compiling failed for " .. dataname .. ", deleting file " .. lucname) + local ok = io.savedata(luaname,input.serialize(data)) + if ok then + input.report("%s saved in %s",dataname,luaname) + if utils.lua.compile(luaname,lucname,false,true) then -- no cleanup but strip + input.report("%s compiled to %s",dataname,lucname) + else + input.report("compiling failed for %s, deleting file %s",dataname,lucname) os.remove(lucname) end else - input.report("unable to save " .. dataname .. " in " .. name..input.luasuffix) + input.report("unable to save %s in %s (access error)",dataname,luaname) end end end -function input.aux.load_data(instance,pathname,dataname,filename,makename) -- untested without cache overload +function input.aux.load_data(pathname,dataname,filename,makename) -- untested without cache overload + local instance = input.instance filename = ((not filename or (filename == "")) and dataname) or filename filename = (makename and makename(dataname,filename)) or file.join(pathname,filename) - local blob = loadfile(filename .. input.lucsuffix) or loadfile(filename .. input.luasuffix) + local blob = loadfile(filename .. ".luc") or loadfile(filename .. ".lua") if blob then local data = blob() if data and data.content and data.type == dataname and data.version == input.cacheversion then - input.report("loading",dataname,"for",pathname,"from",filename) + input.report("loading %s for %s from %s",dataname,pathname,filename) instance[dataname][pathname] = data.content else - input.report("skipping",dataname,"for",pathname,"from",filename) + input.report("skipping %s for %s from %s",dataname,pathname,filename) instance[dataname][pathname] = { } instance.loaderror = true end else - input.report("skipping",dataname,"for",pathname,"from",filename) + input.report("skipping %s for %s from %s",dataname,pathname,filename) end end @@ -3636,13 +3770,14 @@ end -- TEXMFBOGUS = 'effe checken of dit werkt', -- } -function input.aux.load_texmfcnf(instance,dataname,pathname) +function input.aux.load_texmfcnf(dataname,pathname) + local instance = input.instance local filename = file.join(pathname,input.luaname) local blob = loadfile(filename) if blob then local data = blob() if data then - input.report("loading","configuration file",filename) + input.report("loading configuration file %s",filename) if true then -- flatten to variable.progname local t = { } @@ -3662,172 +3797,168 @@ function input.aux.load_texmfcnf(instanc instance[dataname][pathname] = data end else - input.report("skipping","configuration file",filename) + input.report("skipping configuration file %s",filename) instance[dataname][pathname] = { } instance.loaderror = true end else - input.report("skipping","configuration file",filename) + input.report("skipping configuration file %s",filename) end end -function input.aux.load_configuration(instance,dname,lname) - input.aux.load_data(instance,dname,'configuration',lname and file.basename(lname)) +function input.aux.load_configuration(dname,lname) + input.aux.load_data(dname,'configuration',lname and file.basename(lname)) end -function input.aux.load_files(instance,tag) - input.aux.load_data(instance,tag,'files') +function input.aux.load_files(tag) + input.aux.load_data(tag,'files') end -function input.resetconfig(instance) +function input.resetconfig() + input.identify_own() + local instance = input.instance instance.configuration, instance.setup, instance.order, instance.loaderror = { }, { }, { }, false end -function input.loadnewconfig(instance) +function input.loadnewconfig() + local instance = input.instance for _, cnf in ipairs(instance.luafiles) do local dname = file.dirname(cnf) - input.aux.load_texmfcnf(instance,'setup',dname) + input.aux.load_texmfcnf('setup',dname) instance.order[#instance.order+1] = instance.setup[dname] if instance.loaderror then break end end end -function input.loadoldconfig(instance) +function input.loadoldconfig() + local instance = input.instance if not instance.renewcache then for _, cnf in ipairs(instance.cnffiles) do local dname = file.dirname(cnf) - input.aux.load_configuration(instance,dname) + input.aux.load_configuration(dname) instance.order[#instance.order+1] = instance.configuration[dname] if instance.loaderror then break end end end - input.joinconfig(instance) + input.joinconfig() end -function input.expand_variables(instance) - instance.expansions = { } ---~ instance.environment['SELFAUTOPARENT'] = instance.environment['SELFAUTOPARENT'] or instance.rootpath - if instance.engine ~= "" then instance.environment['engine'] = instance.engine end - if instance.progname ~= "" then instance.environment['progname'] = instance.progname end - for k,v in pairs(instance.environment) do +function input.expand_variables() + local instance = input.instance + local expansions, environment, variables = { }, instance.environment, instance.variables + local env = input.env + instance.expansions = expansions + if instance.engine ~= "" then environment['engine'] = instance.engine end + if instance.progname ~= "" then environment['progname'] = instance.progname end + for k,v in pairs(environment) do local a, b = k:match("^(%a+)%_(.*)%s*$") if a and b then - instance.expansions[a..'.'..b] = v + expansions[a..'.'..b] = v else - instance.expansions[k] = v + expansions[k] = v end end - for k,v in pairs(instance.environment) do -- move environment to expansions - if not instance.expansions[k] then instance.expansions[k] = v end + for k,v in pairs(environment) do -- move environment to expansions + if not expansions[k] then expansions[k] = v end end - for k,v in pairs(instance.variables) do -- move variables to expansions - if not instance.expansions[k] then instance.expansions[k] = v end + for k,v in pairs(variables) do -- move variables to expansions + if not expansions[k] then expansions[k] = v end end while true do local busy = false - for k,v in pairs(instance.expansions) do + for k,v in pairs(expansions) do local s, n = v:gsub("%$([%a%d%_%-]+)", function(a) busy = true - return instance.expansions[a] or input.env(instance,a) + return expansions[a] or env(a) end) local s, m = s:gsub("%$%{([%a%d%_%-]+)%}", function(a) busy = true - return instance.expansions[a] or input.env(instance,a) + return expansions[a] or env(a) end) if n > 0 or m > 0 then - instance.expansions[k]= s + expansions[k]= s end end if not busy then break end end - local homedir = - instance.environment[(os.type == "windows" and 'USERPROFILE') or 'HOME'] or '~' - for k,v in pairs(instance.expansions) do - v = v:gsub("^~", homedir) - instance.expansions[k] = v:gsub("\\", '/') + for k,v in pairs(expansions) do + expansions[k] = v:gsub("\\", '/') end end -function input.aux.expand_vars(instance,lst) -- simple vars +function input.aux.expand_vars(lst) -- simple vars + local instance = input.instance + local variables, env = instance.variables, input.env for k,v in pairs(lst) do lst[k] = v:gsub("%$([%a%d%_%-]+)", function(a) - return instance.variables[a] or input.env(instance,a) + return variables[a] or env(a) end) end end -function input.aux.expanded_var(instance,var) -- simple vars +function input.aux.expanded_var(var) -- simple vars + local instance = input.instance return var:gsub("%$([%a%d%_%-]+)", function(a) - return instance.variables[a] or input.env(instance,a) + return instance.variables[a] or input.env(a) end) end -function input.aux.entry(instance,entries,name) +function input.aux.entry(entries,name) if name and (name ~= "") then + local instance = input.instance name = name:gsub('%$','') local result = entries[name..'.'..instance.progname] or entries[name] if result then return result else - result = input.env(instance,name) + result = input.env(name) if result then instance.variables[name] = result - input.expand_variables(instance) + input.expand_variables() return instance.expansions[name] or "" end end end return "" end -function input.variable(instance,name) - return input.aux.entry(instance,instance.variables,name) +function input.variable(name) + return input.aux.entry(input.instance.variables,name) end -function input.expansion(instance,name) - return input.aux.entry(instance,instance.expansions,name) +function input.expansion(name) + return input.aux.entry(input.instance.expansions,name) end -function input.aux.is_entry(instance,entries,name) +function input.aux.is_entry(entries,name) if name and name ~= "" then name = name:gsub('%$','') - return (entries[name..'.'..instance.progname] or entries[name]) ~= nil + return (entries[name..'.'..input.instance.progname] or entries[name]) ~= nil else return false end end -function input.is_variable(instance,name) - return input.aux.is_entry(instance,instance.variables,name) -end -function input.is_expansion(instance,name) - return input.aux.is_entry(instance,instance.expansions,name) +function input.is_variable(name) + return input.aux.is_entry(input.instance.variables,name) end -function input.simplified_list(str) - if type(str) == 'table' then - return str -- troubles ; ipv , in texmf - elseif str == '' then - return { } - else - local t = { } - for _,v in ipairs(string.splitchr(str:gsub("^\{(.+)\}$","%1"),",")) do - t[#t+1] = (v:gsub("^[%!]*(.+)[%/\\]*$","%1")) - end - return t - end +function input.is_expansion(name) + return input.aux.is_entry(input.instance.expansions,name) end -function input.unexpanded_path_list(instance,str) - local pth = input.variable(instance,str) +function input.unexpanded_path_list(str) + local pth = input.variable(str) local lst = input.split_path(pth) - return input.aux.expanded_path(instance,lst) + return input.aux.expanded_path(lst) end -function input.unexpanded_path(instance,str) - return file.join_path(input.unexpanded_path_list(instance,str)) + +function input.unexpanded_path(str) + return file.join_path(input.unexpanded_path_list(str)) end do local done = { } - function input.reset_extra_path(instance) + function input.reset_extra_path() + local instance = input.instance local ep = instance.extra_paths if not ep then ep, done = { }, { } @@ -3837,7 +3968,8 @@ do end end - function input.register_extra_path(instance,paths,subpaths) + function input.register_extra_path(paths,subpaths) + local instance = input.instance local ep = instance.extra_paths or { } local n = #ep if paths and paths ~= "" then @@ -3882,7 +4014,8 @@ do end -function input.expanded_path_list(instance,str) +function input.expanded_path_list(str) + local instance = input.instance local function made_list(list) local ep = instance.extra_paths if not ep or #ep == 0 then @@ -3923,39 +4056,41 @@ function input.expanded_path_list(instan -- engine+progname hash str = str:gsub("%$","") if not instance.lists[str] then -- cached - local lst = made_list(input.split_path(input.expansion(instance,str))) - instance.lists[str] = input.aux.expanded_path(instance,lst) + local lst = made_list(input.split_path(input.expansion(str))) + instance.lists[str] = input.aux.expanded_path(lst) end return instance.lists[str] else - local lst = input.split_path(input.expansion(instance,str)) - return made_list(input.aux.expanded_path(instance,lst)) + local lst = input.split_path(input.expansion(str)) + return made_list(input.aux.expanded_path(lst)) end end -function input.expand_path(instance,str) - return file.join_path(input.expanded_path_list(instance,str)) + +function input.clean_path_list(str) + local t = input.expanded_path_list(str) + if t then + for i=1,#t do + t[i] = file.collapse_path(input.clean_path(t[i])) + end + end + return t end ---~ function input.first_writable_path(instance,name) ---~ for _,v in pairs(input.expanded_path_list(instance,name)) do ---~ if file.is_writable(file.join(v,'luatex-cache.tmp')) then ---~ return v ---~ end ---~ end ---~ return "." ---~ end +function input.expand_path(str) + return file.join_path(input.expanded_path_list(str)) +end -function input.expanded_path_list_from_var(instance,str) -- brrr +function input.expanded_path_list_from_var(str) -- brrr local tmp = input.var_of_format_or_suffix(str:gsub("%$","")) if tmp ~= "" then - return input.expanded_path_list(instance,str) + return input.expanded_path_list(str) else - return input.expanded_path_list(instance,tmp) + return input.expanded_path_list(tmp) end end -function input.expand_path_from_var(instance,str) - return file.join_path(input.expanded_path_list_from_var(instance,str)) +function input.expand_path_from_var(str) + return file.join_path(input.expanded_path_list_from_var(str)) end function input.format_of_var(str) @@ -3985,9 +4120,9 @@ function input.var_of_format_or_suffix(s return '' end -function input.expand_braces(instance,str) -- output variable and brace expansion of STRING - local ori = input.variable(instance,str) - local pth = input.aux.expanded_path(instance,input.split_path(ori)) +function input.expand_braces(str) -- output variable and brace expansion of STRING + local ori = input.variable(str) + local pth = input.aux.expanded_path(input.split_path(ori)) return file.join_path(pth) end @@ -4002,6 +4137,7 @@ end -- {a,b,c/{p,q,r}/d/{x,y,z}//} -- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}} -- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}} +-- {$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c} -- this one is better and faster, but it took me a while to realize -- that this kind of replacement is cleaner than messy parsing and @@ -4010,19 +4146,19 @@ end -- work that well; the parsing is ok, but dealing with the resulting -- table is a pain because we need to work inside-out recursively --- get rid of piecewise here, just a gmatch is ok - function input.aux.splitpathexpr(str, t, validate) -- no need for optimization, only called a few times, we can use lpeg for the sub t = t or { } - local concat = table.concat + str = str:gsub(",}",",@}") + str = str:gsub("{,","{@,") + -- str = "@" .. str .. "@" while true do local done = false while true do local ok = false - str = str:gsub("([^{},]+){([^{}]-)}", function(a,b) + str = str:gsub("([^{},]+){([^{}]+)}", function(a,b) local t = { } - b:piecewise(",", function(s) t[#t+1] = a .. s end) + for s in b:gmatch("[^,]+") do t[#t+1] = a .. s end ok, done = true, true return "{" .. concat(t,",") .. "}" end) @@ -4030,9 +4166,9 @@ function input.aux.splitpathexpr(str, t, end while true do local ok = false - str = str:gsub("{([^{}]-)}([^{},]+)", function(a,b) + str = str:gsub("{([^{}]+)}([^{},]+)", function(a,b) local t = { } - a:piecewise(",", function(s) t[#t+1] = s .. b end) + for s in a:gmatch("[^,]+") do t[#t+1] = s .. b end ok, done = true, true return "{" .. concat(t,",") .. "}" end) @@ -4040,50 +4176,41 @@ function input.aux.splitpathexpr(str, t, end while true do local ok = false - str = str:gsub("([,{]){([^{}]+)}([,}])", function(a,b,c) + str = str:gsub("{([^{}]+)}{([^{}]+)}", function(a,b) + local t = { } + for sa in a:gmatch("[^,]+") do + for sb in b:gmatch("[^,]+") do + t[#t+1] = sa .. sb + end + end ok, done = true, true - return a .. b .. c + return "{" .. concat(t,",") .. "}" end) if not ok then break end end - if not done then break end - end - while true do - local ok = false - str = str:gsub("{([^{}]-)}{([^{}]-)}", function(a,b) - local t = { } - a:piecewise(",", function(sa) - b:piecewise(",", function(sb) - t[#t+1] = sa .. sb - end) - end) - ok = true - return "{" .. concat(t,",") .. "}" - end) - if not ok then break end - end - while true do - local ok = false - str = str:gsub("{([^{}]-)}", function(a) - ok = true - return a + str = str:gsub("({[^{}]*){([^{}]+)}([^{}]*})", function(a,b,c) + done = true + return a .. b.. c end) - if not ok then break end + if not done then break end end + str = str:gsub("[{}]", "") + str = str:gsub("@","") if validate then - str:piecewise(",", function(s) + for s in str:gmatch("[^,]+") do s = validate(s) if s then t[#t+1] = s end - end) + end else - str:piecewise(",", function(s) + for s in str:gmatch("[^,]+") do t[#t+1] = s - end) + end end return t end -function input.aux.expanded_path(instance,pathlist) -- maybe not a list, just a path +function input.aux.expanded_path(pathlist) -- maybe not a list, just a path + local instance = input.instance -- a previous version fed back into pathlist local newlist, ok = { }, false for _,v in ipairs(pathlist) do @@ -4115,17 +4242,16 @@ input.is_readable = { } function input.aux.is_readable(readable, name) if input.trace > 2 then if readable then - input.logger("+ readable", name) + input.logger("+ readable: %s",name) else - input.logger("- readable", name) + input.logger("- readable: %s", name) end end return readable end function input.is_readable.file(name) - -- return input.aux.is_readable(file.is_readable(name), name) - return input.aux.is_readable(input.aux.is_file(name), name) + return input.aux.is_readable(lfs.isfile(name), name) end input.is_readable.tex = input.is_readable.file @@ -4133,12 +4259,13 @@ input.is_readable.tex = input.is_readabl -- name -- name/name -function input.aux.collect_files(instance,names) +function input.aux.collect_files(names) + local instance = input.instance local filelist = { } for _, fname in pairs(names) do if fname then if input.trace > 2 then - input.logger("? blobpath asked",fname) + input.logger("? blobpath asked: %s",fname) end local bname = file.basename(fname) local dname = file.dirname(fname) @@ -4152,7 +4279,7 @@ function input.aux.collect_files(instanc local files = blobpath and instance.files[blobpath] if files then if input.trace > 2 then - input.logger('? blobpath do',blobpath .. " (" .. bname ..")") + input.logger('? blobpath do: %s (%s)',blobpath,bname) end local blobfile = files[bname] if not blobfile then @@ -4185,7 +4312,7 @@ function input.aux.collect_files(instanc end end elseif input.trace > 1 then - input.logger('! blobpath no',blobpath .. " (" .. bname ..")" ) + input.logger('! blobpath no: %s (%s)',blobpath,bname) end end end @@ -4240,15 +4367,17 @@ do end -function input.aux.register_in_trees(instance,name) +function input.aux.register_in_trees(name) if not name:find("^%.") then + local instance = input.instance instance.foundintrees[name] = (instance.foundintrees[name] or 0) + 1 -- maybe only one end end -- split the next one up, better for jit -function input.aux.find_file(instance,filename) -- todo : plugin (scanners, checkers etc) +function input.aux.find_file(filename) -- todo : plugin (scanners, checkers etc) + local instance = input.instance local result = { } local stamp = nil filename = input.normalize_name(filename) -- elsewhere @@ -4257,16 +4386,22 @@ function input.aux.find_file(instance,fi if instance.remember then stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format if instance.found[stamp] then - input.logger('! remembered', filename) + if input.trace > 0 then + input.logger('! remembered: %s',filename) + end return instance.found[stamp] end end if filename:find('%*') then - input.logger('! wildcard', filename) - result = input.find_wildcard_files(instance,filename) + if input.trace > 0 then + input.logger('! wildcard: %s', filename) + end + result = input.find_wildcard_files(filename) elseif input.aux.qualified_path(filename) then if input.is_readable.file(filename) then - input.logger('! qualified', filename) + if input.trace > 0 then + input.logger('! qualified: %s', filename) + end result = { filename } else local forcedname, ok = "", false @@ -4274,22 +4409,26 @@ function input.aux.find_file(instance,fi if instance.format == "" then forcedname = filename .. ".tex" if input.is_readable.file(forcedname) then - input.logger('! no suffix, forcing standard filetype tex') + if input.trace > 0 then + input.logger('! no suffix, forcing standard filetype: tex') + end result, ok = { forcedname }, true end else for _, s in pairs(input.suffixes_of_format(instance.format)) do forcedname = filename .. "." .. s if input.is_readable.file(forcedname) then - input.logger('! no suffix, forcing format filetype', s) + if input.trace > 0 then + input.logger('! no suffix, forcing format filetype: %s', s) + end result, ok = { forcedname }, true break end end end end - if not ok then - input.logger('? qualified', filename) + if not ok and input.trace > 0 then + input.logger('? qualified: %s', filename) end end else @@ -4307,10 +4446,14 @@ function input.aux.find_file(instance,fi local forcedname = filename .. '.tex' wantedfiles[#wantedfiles+1] = forcedname filetype = input.format_of_suffix(forcedname) - input.logger('! forcing filetype',filetype) + if input.trace > 0 then + input.logger('! forcing filetype: %s',filetype) + end else filetype = input.format_of_suffix(filename) - input.logger('! using suffix based filetype',filetype) + if input.trace > 0 then + input.logger('! using suffix based filetype: %s',filetype) + end end else if ext == "" then @@ -4319,16 +4462,18 @@ function input.aux.find_file(instance,fi end end filetype = instance.format - input.logger('! using given filetype',filetype) + if input.trace > 0 then + input.logger('! using given filetype: %s',filetype) + end end local typespec = input.variable_of_format(filetype) - local pathlist = input.expanded_path_list(instance,typespec) + local pathlist = input.expanded_path_list(typespec) if not pathlist or #pathlist == 0 then -- no pathlist, access check only / todo == wildcard if input.trace > 2 then - input.logger('? filename',filename) - input.logger('? filetype',filetype or '?') - input.logger('? wanted files',table.concat(wantedfiles," | ")) + input.logger('? filename: %s',filename) + input.logger('? filetype: %s',filetype or '?') + input.logger('? wanted files: %s',concat(wantedfiles," | ")) end for _, fname in pairs(wantedfiles) do if fname and input.is_readable.file(fname) then @@ -4338,7 +4483,7 @@ function input.aux.find_file(instance,fi end end -- this is actually 'other text files' or 'any' or 'whatever' - local filelist = input.aux.collect_files(instance,wantedfiles) + local filelist = input.aux.collect_files(wantedfiles) local fl = filelist and filelist[1] if fl then filename = fl[3] @@ -4347,12 +4492,12 @@ function input.aux.find_file(instance,fi end else -- list search - local filelist = input.aux.collect_files(instance,wantedfiles) + local filelist = input.aux.collect_files(wantedfiles) local doscan, recurse if input.trace > 2 then - input.logger('? filename',filename) - -- if pathlist then input.logger('? path list',table.concat(pathlist," | ")) end - -- if filelist then input.logger('? file list',table.concat(filelist," | ")) end + input.logger('? filename: %s',filename) + -- if pathlist then input.logger('? path list: %s',concat(pathlist," | ")) end + -- if filelist then input.logger('? file list: %s',concat(filelist," | ")) end end -- a bit messy ... esp the doscan setting here for _, path in pairs(pathlist) do @@ -4373,11 +4518,11 @@ function input.aux.find_file(instance,fi if f:find(expr) then -- input.debug('T',' '..f) if input.trace > 2 then - input.logger('= found in hash',f) + input.logger('= found in hash: %s',f) end --- todo, test for readable result[#result+1] = fl[3] - input.aux.register_in_trees(instance,f) -- for tracing used files + input.aux.register_in_trees(f) -- for tracing used files done = true if not instance.allresults then break end else @@ -4391,12 +4536,12 @@ function input.aux.find_file(instance,fi local pname = pathname:gsub("%.%*$",'') if not pname:find("%*") then local ppname = pname:gsub("/+$","") - if input.aux.can_be_dir(instance,ppname) then + if input.aux.can_be_dir(ppname) then for _, w in pairs(wantedfiles) do local fname = file.join(ppname,w) if input.is_readable.file(fname) then if input.trace > 2 then - input.logger('= found by scanning',fname) + input.logger('= found by scanning: %s',fname) end result[#result+1] = fname done = true @@ -4425,40 +4570,29 @@ function input.aux.find_file(instance,fi return result end -input.aux._find_file_ = input.aux.find_file +input.aux._find_file_ = input.aux.find_file -- frozen variant -function input.aux.find_file(instance,filename) -- maybe make a lowres cache too - local result = input.aux._find_file_(instance,filename) +function input.aux.find_file(filename) -- maybe make a lowres cache too + local result = input.aux._find_file_(filename) if #result == 0 then local lowered = filename:lower() if filename ~= lowered then - return input.aux._find_file_(instance,lowered) + return input.aux._find_file_(lowered) end end return result end -if lfs and lfs.isfile then - input.aux.is_file = lfs.isfile -- to be done: use this -else - input.aux.is_file = file.is_readable -end - -if lfs and lfs.isdir then - function input.aux.can_be_dir(instance,name) - if not instance.fakepaths[name] then - if lfs.isdir(name) then - instance.fakepaths[name] = 1 -- directory - else - instance.fakepaths[name] = 2 -- no directory - end +function input.aux.can_be_dir(name) + local instance = input.instance + if not instance.fakepaths[name] then + if lfs.isdir(name) then + instance.fakepaths[name] = 1 -- directory + else + instance.fakepaths[name] = 2 -- no directory end - return (instance.fakepaths[name] == 1) - end -else - function input.aux.can_be_dir() - return true end + return (instance.fakepaths[name] == 1) end if not input.concatinators then input.concatinators = { } end @@ -4466,7 +4600,8 @@ if not input.concatinators then input.c input.concatinators.tex = file.join input.concatinators.file = input.concatinators.tex -function input.find_files(instance,filename,filetype,mustexist) +function input.find_files(filename,filetype,mustexist) + local instance = input.instance if type(mustexist) == boolean then -- all set elseif type(filetype) == 'boolean' then @@ -4475,16 +4610,17 @@ function input.find_files(instance,filen filetype, mustexist = nil, false end instance.format = filetype or '' - local t = input.aux.find_file(instance,filename,true) + local t = input.aux.find_file(filename,true) instance.format = '' return t end -function input.find_file(instance,filename,filetype,mustexist) - return (input.find_files(instance,filename,filetype,mustexist)[1] or "") +function input.find_file(filename,filetype,mustexist) + return (input.find_files(filename,filetype,mustexist)[1] or "") end -function input.find_given_files(instance,filename) +function input.find_given_files(filename) + local instance = input.instance local bname, result = file.basename(filename), { } for k, hash in ipairs(instance.hashes) do local files = instance.files[hash.tag] @@ -4512,11 +4648,12 @@ function input.find_given_files(instance return result end -function input.find_given_file(instance,filename) - return (input.find_given_files(instance,filename)[1] or "") +function input.find_given_file(filename) + return (input.find_given_files(filename)[1] or "") end -function input.find_wildcard_files(instance,filename) -- todo: remap: +function input.find_wildcard_files(filename) -- todo: remap: + local instance = input.instance local result = { } local bname, dname = file.basename(filename), file.dirname(filename) local path = dname:gsub("^*/","") @@ -4569,16 +4706,19 @@ function input.find_wildcard_files(insta if done and not allresults then break end end end + -- we can consider also searching the paths not in the database, but then + -- we end up with a messy search (all // in all path specs) return result end -function input.find_wildcard_file(instance,filename) - return (input.find_wildcard_files(instance,filename)[1] or "") +function input.find_wildcard_file(filename) + return (input.find_wildcard_files(filename)[1] or "") end -- main user functions -function input.save_used_files_in_trees(instance, filename,jobname) +function input.save_used_files_in_trees(filename,jobname) + local instance = input.instance if not filename then filename = 'luatex.jlg' end local f = io.open(filename,'w') if f then @@ -4588,7 +4728,7 @@ function input.save_used_files_in_trees( f:write("\t<rl:name>" .. jobname .. "</rl:name>\n") end f:write("\t<rl:files>\n") - for _,v in pairs(table.sortedkeys(instance.foundintrees)) do + for _,v in pairs(sorted(instance.foundintrees)) do -- ipairs f:write("\t\t<rl:file n='" .. instance.foundintrees[v] .. "'>" .. v .. "</rl:file>\n") end f:write("\t</rl:files>\n") @@ -4597,24 +4737,24 @@ function input.save_used_files_in_trees( end end -function input.automount(instance) +function input.automount() -- implemented later end -function input.load(instance) - input.starttiming(instance) - input.resetconfig(instance) - input.identify_cnf(instance) - input.load_lua(instance) - input.expand_variables(instance) - input.load_cnf(instance) - input.expand_variables(instance) - input.load_hash(instance) - input.automount(instance) - input.stoptiming(instance) +function input.load() + input.starttiming(input.instance) + input.resetconfig() + input.identify_cnf() + input.load_lua() + input.expand_variables() + input.load_cnf() + input.expand_variables() + input.load_hash() + input.automount() + input.stoptiming(input.instance) end -function input.for_files(instance, command, files, filetype, mustexist) +function input.for_files(command, files, filetype, mustexist) if files and #files > 0 then local function report(str) if input.verbose then @@ -4627,7 +4767,7 @@ function input.for_files(instance, comma report('') end for _, file in pairs(files) do - local result = command(instance,file,filetype,mustexist) + local result = command(file,filetype,mustexist) if type(result) == 'string' then report(result) else @@ -4641,14 +4781,11 @@ end -- strtab -function input.var_value(instance,str) -- output the value of variable $STRING. - return input.variable(instance,str) -end -function input.expand_var(instance,str) -- output variable expansion of STRING. - return input.expansion(instance,str) -end -function input.show_path(instance,str) -- output search path for file type NAME - return file.join_path(input.expanded_path_list(instance,input.format_of_var(str))) +input.var_value = input.variable -- output the value of variable $STRING. +input.expand_var = input.expansion -- output variable expansion of STRING. + +function input.show_path(str) -- output search path for file type NAME + return file.join_path(input.expanded_path_list(input.format_of_var(str))) end -- input.find_file(filename) @@ -4697,56 +4834,58 @@ function table.sequenced(t,sep) -- temp for k, v in pairs(t) do s[#s+1] = k .. "=" .. v end - return table.concat(s, sep or " | ") + return concat(s, sep or " | ") end -function input.methodhandler(what, instance, filename, filetype) -- ... +function input.methodhandler(what, filename, filetype) -- ... local specification = (type(filename) == "string" and input.splitmethod(filename)) or filename -- no or { }, let it bomb local scheme = specification.scheme if input[what][scheme] then - input.logger('= handler',specification.original .." -> " .. what .. " -> " .. table.sequenced(specification)) - return input[what][scheme](instance,filename,filetype) -- todo: specification + if input.trace > 0 then + input.logger('= handler: %s -> %s -> %s',specification.original,what,table.sequenced(specification)) + end + return input[what][scheme](filename,filetype) -- todo: specification else - return input[what].tex(instance,filename,filetype) -- todo: specification + return input[what].tex(filename,filetype) -- todo: specification end end -- also inside next test? -function input.findtexfile(instance, filename, filetype) - return input.methodhandler('finders',instance, input.normalize_name(filename), filetype) +function input.findtexfile(filename, filetype) + return input.methodhandler('finders',input.normalize_name(filename), filetype) end -function input.opentexfile(instance,filename) - return input.methodhandler('openers',instance, input.normalize_name(filename)) +function input.opentexfile(filename) + return input.methodhandler('openers',input.normalize_name(filename)) end -function input.findbinfile(instance, filename, filetype) - return input.methodhandler('finders',instance, input.normalize_name(filename), filetype) +function input.findbinfile(filename, filetype) + return input.methodhandler('finders',input.normalize_name(filename), filetype) end -function input.openbinfile(instance,filename) - return input.methodhandler('loaders',instance, input.normalize_name(filename)) +function input.openbinfile(filename) + return input.methodhandler('loaders',input.normalize_name(filename)) end -function input.loadbinfile(instance, filename, filetype) - local fname = input.findbinfile(instance, input.normalize_name(filename), filetype) +function input.loadbinfile(filename, filetype) + local fname = input.findbinfile(input.normalize_name(filename), filetype) if fname and fname ~= "" then - return input.openbinfile(instance,fname) + return input.openbinfile(fname) else return unpack(input.loaders.notfound) end end -function input.texdatablob(instance, filename, filetype) - local ok, data, size = input.loadbinfile(instance, filename, filetype) +function input.texdatablob(filename, filetype) + local ok, data, size = input.loadbinfile(filename, filetype) return data or "" end input.loadtexfile = input.texdatablob -function input.openfile(filename) -- brrr texmf.instance here / todo ! ! ! ! ! - local fullname = input.findtexfile(texmf.instance, filename) +function input.openfile(filename) + local fullname = input.findtexfile(filename) if fullname and (fullname ~= "") then - return input.opentexfile(texmf.instance, fullname) + return input.opentexfile(fullname) else return nil end @@ -4792,16 +4931,18 @@ end -- beware: i need to check where we still need a / on windows: function input.clean_path(str) ---~ return (((str:gsub("\\","/")):gsub("^!+","")):gsub("//+","//")) if str then - return ((str:gsub("\\","/")):gsub("^!+","")) + str = str:gsub("\\","/") + str = str:gsub("^!+","") + str = str:gsub("^~",input.homedir) + return str else return nil end end function input.do_with_path(name,func) - for _, v in pairs(input.expanded_path_list(instance,name)) do + for _, v in pairs(input.expanded_path_list(name)) do func("^"..input.clean_path(v)) end end @@ -4810,7 +4951,8 @@ function input.do_with_var(name,func) func(input.aux.expanded_var(name)) end -function input.with_files(instance,pattern,handle) +function input.with_files(pattern,handle) + local instance = input.instance for _, hash in ipairs(instance.hashes) do local blobpath = hash.tag local blobtype = hash.type @@ -4837,37 +4979,22 @@ function input.with_files(instance,patte end end ---~ function input.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix ---~ newname = file.addsuffix(newname,"lua") ---~ local newscript = input.clean_path(input.find_file(instance, newname)) ---~ local oldscript = input.clean_path(oldname) ---~ input.report("old script", oldscript) ---~ input.report("new script", newscript) ---~ if oldscript ~= newscript and (oldscript:find(file.removesuffix(newname).."$") or oldscript:find(newname.."$")) then ---~ local newdata = io.loaddata(newscript) ---~ if newdata then ---~ input.report("old script content replaced by new content") ---~ io.savedata(oldscript,newdata) ---~ end ---~ end ---~ end - -function input.update_script(instance,oldname,newname) -- oldname -> own.name, not per se a suffix +function input.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix local scriptpath = "scripts/context/lua" newname = file.addsuffix(newname,"lua") local oldscript = input.clean_path(oldname) - input.report("to be replaced old script", oldscript) - local newscripts = input.find_files(instance, newname) or { } + input.report("to be replaced old script %s", oldscript) + local newscripts = input.find_files(newname) or { } if #newscripts == 0 then input.report("unable to locate new script") else for _, newscript in ipairs(newscripts) do newscript = input.clean_path(newscript) - input.report("checking new script", newscript) + input.report("checking new script %s", newscript) if oldscript == newscript then input.report("old and new script are the same") elseif not newscript:find(scriptpath) then - input.report("new script should come from",scriptpath) + input.report("new script should come from %s",scriptpath) elseif not (oldscript:find(file.removesuffix(newname).."$") or oldscript:find(newname.."$")) then input.report("invalid new script name") else @@ -4895,10 +5022,10 @@ do local resolvers = { } - resolvers.environment = function(instance,str) + resolvers.environment = function(str) return input.clean_path(os.getenv(str) or os.getenv(str:upper()) or os.getenv(str:lower()) or "") end - resolvers.relative = function(instance,str,n) + resolvers.relative = function(str,n) if io.exists(str) then -- nothing elseif io.exists("./" .. str) then @@ -4916,16 +5043,16 @@ do end return input.clean_path(str) end - resolvers.locate = function(instance,str) - local fullname = input.find_given_file(instance,str) or "" + resolvers.locate = function(str) + local fullname = input.find_given_file(str) or "" return input.clean_path((fullname ~= "" and fullname) or str) end - resolvers.filename = function(instance,str) - local fullname = input.find_given_file(instance,str) or "" + resolvers.filename = function(str) + local fullname = input.find_given_file(str) or "" return input.clean_path(file.basename((fullname ~= "" and fullname) or str)) end - resolvers.pathname = function(instance,str) - local fullname = input.find_given_file(instance,str) or "" + resolvers.pathname = function(str) + local fullname = input.find_given_file(str) or "" return input.clean_path(file.dirname((fullname ~= "" and fullname) or str)) end @@ -4937,15 +5064,15 @@ do resolvers.file = resolvers.filename resolvers.path = resolvers.pathname - local function resolve(instance,str) + local function resolve(str) if type(str) == "table" then for k, v in pairs(str) do - str[k] = resolve(instance,v) or v + str[k] = resolve(v) or v end elseif str and str ~= "" then - str = str:gsub("([a-z]+):([^ ]+)", function(method,target) + str = str:gsub("([a-z]+):([^ \"\']*)", function(method,target) if resolvers[method] then - return resolvers[method](instance,target) + return resolvers[method](target) else return method .. ":" .. target end @@ -4954,10 +5081,185 @@ do return str end + if os.uname then + for k, v in pairs(os.uname()) do + if not resolvers[k] then + resolvers[k] = function() return v end + end + end + end + input.resolve = resolve end +function input.boolean_variable(str,default) + local b = input.expansion(str) + if b == "" then + return default + else + b = toboolean(b) + return (b == nil and default) or b + end +end + + +if not modules then modules = { } end modules ['luat-log'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +--[[ldx-- +<p>This is a prelude to a more extensive logging module. For the sake +of parsing log files, in addition to the standard logging we will +provide an <l n='xml'/> structured file. Actually, any logging that +is hooked into callbacks will be \XML\ by default.</p> +--ldx]]-- + +-- input.logger -> special tracing, driven by log level (only input) +-- input.report -> goes to terminal, depends on verbose, has banner +-- logs.report -> module specific tracing and reporting, no banner but class + + +input = input or { } +logs = logs or { } + +--[[ldx-- +<p>This looks pretty ugly but we need to speed things up a bit.</p> +--ldx]]-- + +logs.levels = { + ['error'] = 1, + ['warning'] = 2, + ['info'] = 3, + ['debug'] = 4 +} + +logs.functions = { + 'report', 'start', 'stop', 'push', 'pop', 'line', 'direct' +} + +logs.callbacks = { + 'start_page_number', + 'stop_page_number', + 'report_output_pages', + 'report_output_log' +} + +logs.tracers = { +} + +logs.xml = logs.xml or { } +logs.tex = logs.tex or { } + +logs.level = 0 + +local write_nl, write, format = texio.write_nl or print, texio.write or io.write, string.format + +if texlua then + write_nl = print + write = io.write +end + +function logs.xml.report(category,fmt,...) -- new + write_nl(format("<r category='%s'>%s</r>",category,format(fmt,...))) +end +function logs.xml.line(fmt,...) -- new + write_nl(format("<r>%s</r>",format(fmt,...))) +end + +function logs.xml.start() if logs.level > 0 then tw("<%s>" ) end end +function logs.xml.stop () if logs.level > 0 then tw("</%s>") end end +function logs.xml.push () if logs.level > 0 then tw("<!-- ") end end +function logs.xml.pop () if logs.level > 0 then tw(" -->" ) end end + +function logs.tex.report(category,fmt,...) -- new + -- write_nl(format("%s | %s",category,format(fmt,...))) -- arg to format can be tex comment so .. . + write_nl(category .. " | " .. format(fmt,...)) +end +function logs.tex.line(fmt,...) -- new + write_nl(format(fmt,...)) +end + +function logs.set_level(level) + logs.level = logs.levels[level] or level +end + +function logs.set_method(method) + for _, v in pairs(logs.functions) do + logs[v] = logs[method][v] or function() end + end + if callback and input[method] then + for _, cb in pairs(logs.callbacks) do + callback.register(cb, input[method][cb]) + end + end +end + +function logs.xml.start_page_number() + write_nl(format("<p real='%s' page='%s' sub='%s'", tex.count[0], tex.count[1], tex.count[2])) +end + +function logs.xml.stop_page_number() + write("/>") + write_nl("") +end + +function logs.xml.report_output_pages(p,b) + write_nl(format("<v k='pages' v='%s'/>", p)) + write_nl(format("<v k='bytes' v='%s'/>", b)) + write_nl("") +end + +function logs.xml.report_output_log() +end + +function input.logger(...) -- assumes test for input.trace > n + if input.trace > 0 then + logs.report(...) + end +end + +function input.report(fmt,...) + if input.verbose then + logs.report(input.banner or "report",format(fmt,...)) + end +end + +function input.reportlines(str) -- todo: <lines></lines> + for line in str:gmatch("(.-)[\n\r]") do + logs.report(input.banner or "report",line) + end +end + +input.moreinfo = [[ +more information about ConTeXt and the tools that come with it can be found at: + +maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context +webpage : http://www.pragma-ade.nl / http://tex.aanhet.net +wiki : http://contextgarden.net +]] + +function input.help(banner,message) + if not input.verbose then + input.verbose = true + -- input.report(banner,"\n") + end + input.report(banner,"\n") + input.report("") + input.reportlines(message) + if input.moreinfo and input.moreinfo ~= "" then + input.report("") + input.reportlines(input.moreinfo) + end +end + +logs.set_level('error') +logs.set_method('tex') + if not modules then modules = { } end modules ['luat-tmp'] = { version = 1.001, @@ -4983,63 +5285,82 @@ being written at the same time is small. luatools with a recache feature.</p> --ldx]]-- +local format = string.format + caches = caches or { } dir = dir or { } texmf = texmf or { } -caches.path = caches.path or nil -caches.base = caches.base or "luatex-cache" -caches.more = caches.more or "context" -caches.direct = false -- true is faster but may need huge amounts of memory -caches.trace = false -caches.tree = false -caches.paths = caches.paths or nil -caches.force = false - -input.usecache = not toboolean(os.getenv("TEXMFSHARECACHE") or "false",true) -- true - -function caches.temp(instance) - local function checkpath(cachepath) - if not cachepath or cachepath == "" then - return nil - elseif lfs.attributes(cachepath,"mode") == "directory" then -- lfs.isdir(cachepath) then - return cachepath - elseif caches.force or io.ask(string.format("Should I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then - dir.mkdirs(cachepath) - return (lfs.attributes(cachepath,"mode") == "directory") and cachepath - else - return nil +caches.path = caches.path or nil +caches.base = caches.base or "luatex-cache" +caches.more = caches.more or "context" +caches.direct = false -- true is faster but may need huge amounts of memory +caches.trace = false +caches.tree = false +caches.paths = caches.paths or nil +caches.force = false +caches.defaults = { "TEXMFCACHE", "TMPDIR", "TEMPDIR", "TMP", "TEMP", "HOME", "HOMEPATH" } + +function caches.temp() + local cachepath = nil + local function check(list,isenv) + if not cachepath then + for _, v in ipairs(list) do + cachepath = (isenv and (os.env[v] or "")) or v or "" + if cachepath == "" then + -- next + else + cachepath = input.clean_path(cachepath) + if lfs.isdir(cachepath) and file.iswritable(cachepath) then -- lfs.attributes(cachepath,"mode") == "directory" + break + elseif caches.force or io.ask(format("\nShould I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then + dir.mkdirs(cachepath) + if lfs.isdir(cachepath) and file.iswritable(cachepath) then + break + end + end + end + cachepath = nil + end end end - local cachepath = input.expanded_path_list(instance,"TEXMFCACHE") - cachepath = cachepath and #cachepath > 0 and checkpath(cachepath[1]) - if not cachepath then - cachepath = os.getenv("TEXMFCACHE") or os.getenv("HOME") or os.getenv("HOMEPATH") or os.getenv("TMP") or os.getenv("TEMP") or os.getenv("TMPDIR") or nil - cachepath = checkpath(cachepath) - end + check(input.clean_path_list("TEXMFCACHE") or { }) + check(caches.defaults,true) if not cachepath then - print("\nfatal error: there is no valid cache path defined\n") + print("\nfatal error: there is no valid (writable) cache path defined\n") os.exit() - elseif lfs.attributes(cachepath,"mode") ~= "directory" then - print(string.format("\nfatal error: cache path %s is not a directory\n",cachepath)) + elseif not lfs.isdir(cachepath) then -- lfs.attributes(cachepath,"mode") ~= "directory" + print(format("\nfatal error: cache path %s is not a directory\n",cachepath)) os.exit() end - function caches.temp(instance) + cachepath = input.normalize_name(cachepath) + function caches.temp() return cachepath end return cachepath end -function caches.configpath(instance) - return table.concat(instance.cnffiles,";") +function caches.configpath() + return table.concat(input.instance.cnffiles,";") end function caches.hashed(tree) return md5.hex((tree:lower()):gsub("[\\\/]+","/")) end -function caches.treehash(instance) - local tree = caches.configpath(instance) +--~ tracing: + +--~ function caches.hashed(tree) +--~ tree = (tree:lower()):gsub("[\\\/]+","/") +--~ local hash = md5.hex(tree) +--~ if input.verbose then -- temp message +--~ input.report("hashing %s => %s",tree,hash) +--~ end +--~ return hash +--~ end + +function caches.treehash() + local tree = caches.configpath() if not tree or tree == "" then return false else @@ -5047,14 +5368,14 @@ function caches.treehash(instance) end end -function caches.setpath(instance,...) +function caches.setpath(...) if not caches.path then if not caches.path then - caches.path = caches.temp(instance) + caches.path = caches.temp() end caches.path = input.clean_path(caches.path) -- to be sure if lfs then - caches.tree = caches.tree or caches.treehash(instance) + caches.tree = caches.tree or caches.treehash() if caches.tree then caches.path = dir.mkdirs(caches.path,caches.base,caches.more,caches.tree) else @@ -5074,9 +5395,9 @@ function caches.setpath(instance,...) return caches.path end -function caches.definepath(instance,category,subcategory) +function caches.definepath(category,subcategory) return function() - return caches.setpath(instance,category,subcategory) + return caches.setpath(category,subcategory) end end @@ -5099,26 +5420,28 @@ function caches.is_writable(filepath,fil return file.is_writable(tmaname) end -function caches.savedata(filepath,filename,data,raw) -- raw needed for file cache +function caches.savedata(filepath,filename,data,raw) local tmaname, tmcname = caches.setluanames(filepath,filename) local reduce, simplify = true, true if raw then reduce, simplify = false, false end if caches.direct then - file.savedata(tmaname, table.serialize(data,'return',true,true)) + file.savedata(tmaname, table.serialize(data,'return',true,true,false)) -- no hex else - table.tofile (tmaname, data,'return',true,true) -- maybe not the last true + table.tofile(tmaname, data,'return',true,true,false) -- maybe not the last true end - utils.lua.compile(tmaname, tmcname) + local cleanup = input.boolean_variable("PURGECACHE", false) + local strip = input.boolean_variable("LUACSTRIP", true) + utils.lua.compile(tmaname, tmcname, cleanup, strip) end -- here we use the cache for format loading (texconfig.[formatname|jobname]) --~ if tex and texconfig and texconfig.formatname and texconfig.formatname == "" then -if tex and texconfig and (not texconfig.formatname or texconfig.formatname == "") and texmf.instance then +if tex and texconfig and (not texconfig.formatname or texconfig.formatname == "") and input and input.instance then if not texconfig.luaname then texconfig.luaname = "cont-en.lua" end -- or luc - texconfig.formatname = caches.setpath(texmf.instance,"formats") .. "/" .. texconfig.luaname:gsub("%.lu.$",".fmt") + texconfig.formatname = caches.setpath("formats") .. "/" .. texconfig.luaname:gsub("%.lu.$",".fmt") end --[[ldx-- @@ -5141,7 +5464,7 @@ do -- local report local function report(container,tag,name) if caches.trace or containers.trace or container.trace then - logs.report(string.format("%s cache",container.subcategory),string.format("%s: %s",tag,name or 'invalid')) + logs.report(format("%s cache",container.subcategory),"%s: %s",tag,name or 'invalid') end end @@ -5166,7 +5489,7 @@ do -- local report enabled = enabled, version = version or 1.000, trace = false, - path = caches.setpath(texmf.instance,category,subcategory), + path = caches.setpath(category,subcategory), } c[subcategory] = s end @@ -5231,13 +5554,16 @@ end -- reimplement the saver. local save_data = input.aux.save_data +local load_data = input.aux.load_data -input.cachepath = nil +input.cachepath = nil -- public, for tracing +input.usecache = true -- public, for tracing -function input.aux.save_data(instance, dataname, check) - input.cachepath = input.cachepath or caches.definepath(instance,"trees") - save_data(instance, dataname, check, function(cachename,dataname) +function input.aux.save_data(dataname, check) + save_data(dataname, check, function(cachename,dataname) + input.usecache = not toboolean(input.expansion("CACHEINTDS") or "false",true) if input.usecache then + input.cachepath = input.cachepath or caches.definepath("trees") return file.join(input.cachepath(),caches.hashed(cachename)) else return file.join(cachename,dataname) @@ -5245,12 +5571,11 @@ function input.aux.save_data(instance, d end) end -local load_data = input.aux.load_data - -function input.aux.load_data(instance,pathname,dataname,filename) - input.cachepath = input.cachepath or caches.definepath(instance,"trees") - load_data(instance,pathname,dataname,filename,function(dataname,filename) +function input.aux.load_data(pathname,dataname,filename) + load_data(pathname,dataname,filename,function(dataname,filename) + input.usecache = not toboolean(input.expansion("CACHEINTDS") or "false",true) if input.usecache then + input.cachepath = input.cachepath or caches.definepath("trees") return file.join(input.cachepath(),caches.hashed(pathname)) else if not filename or (filename == "") then @@ -5265,13 +5590,13 @@ end input.automounted = input.automounted or { } -function input.automount(instance,usecache) - local mountpaths = input.simplified_list(input.expansion(instance,'TEXMFMOUNT')) +function input.automount(usecache) + local mountpaths = input.clean_path_list(input.expansion('TEXMFMOUNT')) if table.is_empty(mountpaths) and usecache then - mountpaths = { caches.setpath(instance,"mount") } + mountpaths = { caches.setpath("mount") } end if not table.is_empty(mountpaths) then - input.starttiming(instance) + input.starttiming(input.instance) for k, root in pairs(mountpaths) do local f = io.open(root.."/url.tmi") if f then @@ -5280,16 +5605,16 @@ function input.automount(instance,usecac if line:find("^[%%#%-]") then -- or %W -- skip elseif line:find("^zip://") then - input.report("mounting",line) + input.report("mounting %s",line) table.insert(input.automounted,line) - input.usezipfile(instance,line) + input.usezipfile(line) end end end f:close() end end - input.stoptiming(instance) + input.stoptiming(input.instance) end end @@ -5300,7 +5625,7 @@ input.storage.data = { } input.storage.min = 0 -- 500 input.storage.max = input.storage.min - 1 input.storage.trace = false -- true -input.storage.done = 0 +input.storage.done = input.storage.done or 0 input.storage.evaluators = { } -- (evaluate,message,names) @@ -5338,17 +5663,17 @@ function input.storage.dump() else name = str end - initialize = string.format("%s %s = %s or {} ", initialize, name, name) + initialize = format("%s %s = %s or {} ", initialize, name, name) end if evaluate then finalize = "input.storage.evaluate(" .. name .. ")" end input.storage.max = input.storage.max + 1 if input.storage.trace then - logs.report('storage',string.format('saving %s in slot %s',message,input.storage.max)) + logs.report('storage','saving %s in slot %s',message,input.storage.max) code = initialize .. - string.format("logs.report('storage','restoring %s from slot %s') ",message,input.storage.max) .. + format("logs.report('storage','restoring %s from slot %s') ",message,input.storage.max) .. table.serialize(original,name) .. finalize else @@ -5358,6 +5683,8 @@ function input.storage.dump() end end +-- we also need to count at generation time (nicer for message) + if lua.bytecode then -- from 0 upwards local i = input.storage.min while lua.bytecode[i] do @@ -5377,6 +5704,8 @@ end if not versions then versions = { } end versions['luat-zip'] = 1.001 +local format = string.format + if zip and input then zip.supported = true else @@ -5408,7 +5737,7 @@ else zip.archives = { } zip.registeredfiles = { } - function zip.openarchive(instance,name) + function zip.openarchive(name) if not name or name == "" then return nil else @@ -5416,7 +5745,7 @@ else if arch then return arch else - local full = input.find_file(instance,name) or "" + local full = input.find_file(name) or "" local arch = (full ~= "" and zip.open(full)) or false zip.archives[name] = arch return arch @@ -5424,7 +5753,7 @@ else end end - function zip.closearchive(instance,name) + function zip.closearchive(name) if not name or name == "" and zip.archives[name] then zip.close(zip.archives[name]) zip.archives[name] = nil @@ -5435,20 +5764,22 @@ else -- zip:///texmf.zip?tree=/tex/texmf-local -- zip:///texmf-mine.zip?tree=/tex/texmf-projects - function input.locators.zip(instance,specification) -- where is this used? startup zips (untested) + function input.locators.zip(specification) -- where is this used? startup zips (untested) specification = input.splitmethod(specification) local zipfile = specification.path - local zfile = zip.openarchive(instance,name) -- tricky, could be in to be initialized tree - if zfile then - input.logger('! zip locator', specification.original ..' found') - else - input.logger('? zip locator', specification.original ..' not found') + local zfile = zip.openarchive(name) -- tricky, could be in to be initialized tree + if input.trace > 0 then + if zfile then + input.logger('! zip locator, found: %s',specification.original) + else + input.logger('? zip locator, not found: %s',specification.original) + end end end - function input.hashers.zip(instance,tag,name) - input.report("loading zip file",name,"as",tag) - input.usezipfile(instance,tag .."?tree=" .. name) + function input.hashers.zip(tag,name) + input.report("loading zip file %s as %s",name,tag) + input.usezipfile(tag .."?tree=" .. name) end function input.concatinators.zip(tag,path,name) @@ -5463,101 +5794,124 @@ else return true end - function input.finders.zip(instance,specification,filetype) + function input.finders.zip(specification,filetype) specification = input.splitmethod(specification) if specification.path then local q = url.query(specification.query) if q.name then - local zfile = zip.openarchive(instance,specification.path) + local zfile = zip.openarchive(specification.path) if zfile then - input.logger('! zip finder',specification.path) + if input.trace > 0 then + input.logger('! zip finder, path: %s',specification.path) + end local dfile = zfile:open(q.name) if dfile then dfile = zfile:close() - input.logger('+ zip finder',q.name) + if input.trace > 0 then + input.logger('+ zip finder, name: %s',q.name) + end return specification.original end - else - input.logger('? zip finder',specification.path) + elseif input.trace > 0 then + input.logger('? zip finder, path %s',specification.path) end end end - input.logger('- zip finder',filename) + if input.trace > 0 then + input.logger('- zip finder, name: %s',filename) + end return unpack(input.finders.notfound) end - function input.openers.zip(instance,specification) + function input.openers.zip(specification) local zipspecification = input.splitmethod(specification) if zipspecification.path then local q = url.query(zipspecification.query) if q.name then - local zfile = zip.openarchive(instance,zipspecification.path) + local zfile = zip.openarchive(zipspecification.path) if zfile then - input.logger('+ zip starter',zipspecification.path) + if input.trace > 0 then + input.logger('+ zip starter, path: %s',zipspecification.path) + end local dfile = zfile:open(q.name) if dfile then input.show_open(specification) return input.openers.text_opener(specification,dfile,'zip') end - else - input.logger('- zip starter',zipspecification.path) + elseif input.trace > 0 then + input.logger('- zip starter, path %s',zipspecification.path) end end end - input.logger('- zip opener',filename) + if input.trace > 0 then + input.logger('- zip opener, name: %s',filename) + end return unpack(input.openers.notfound) end - function input.loaders.zip(instance,specification) + function input.loaders.zip(specification) specification = input.splitmethod(specification) if specification.path then local q = url.query(specification.query) if q.name then - local zfile = zip.openarchive(instance,specification.path) + local zfile = zip.openarchive(specification.path) if zfile then - input.logger('+ zip starter',specification.path) + if input.trace > 0 then + input.logger('+ zip starter, path: %s',specification.path) + end local dfile = zfile:open(q.name) if dfile then input.show_load(filename) - input.logger('+ zip loader',filename) + if input.trace > 0 then + input.logger('+ zip loader, name: %s',filename) + end local s = dfile:read("*all") dfile:close() return true, s, #s end - else - input.logger('- zip starter',specification.path) + elseif input.trace > 0 then + input.logger('- zip starter, path: %s',specification.path) end end end - input.logger('- zip loader',filename) + if input.trace > 0 then + input.logger('- zip loader, name: %s',filename) + end return unpack(input.openers.notfound) end -- zip:///somefile.zip -- zip:///somefile.zip?tree=texmf-local -> mount - function input.usezipfile(instance,zipname) + function input.usezipfile(zipname) zipname = validzip(zipname) - input.logger('! zip use','file '..zipname) + if input.trace > 0 then + input.logger('! zip use, file: %s',zipname) + end local specification = input.splitmethod(zipname) local zipfile = specification.path if zipfile and not zip.registeredfiles[zipname] then local tree = url.query(specification.query).tree or "" - input.logger('! zip register','file '..zipname) - local z = zip.openarchive(instance,zipfile) + if input.trace > 0 then + input.logger('! zip register, file: %s',zipname) + end + local z = zip.openarchive(zipfile) if z then - input.logger("= zipfile","registering "..zipname) + local instance = input.instance + if input.trace > 0 then + input.logger("= zipfile, registering: %s",zipname) + end input.starttiming(instance) - input.aux.prepend_hash(instance,'zip',zipname,zipfile) - input.aux.extend_texmf_var(instance,zipname) -- resets hashes too + input.aux.prepend_hash('zip',zipname,zipfile) + input.aux.extend_texmf_var(zipname) -- resets hashes too zip.registeredfiles[zipname] = z instance.files[zipname] = input.aux.register_zip_file(z,tree or "") input.stoptiming(instance) - else - input.logger("? zipfile","unknown "..zipname) + elseif input.trace > 0 then + input.logger("? zipfile, unknown: %s",zipname) end - else - input.logger('! zip register','no file '..zipname) + elseif input.trace > 0 then + input.logger('! zip register, no file: %s',zipname) end end @@ -5568,7 +5922,9 @@ else else filter = "^"..tree.."/(.+)/(.-)$" end - input.logger('= zip filter',filter) + if input.trace > 0 then + input.logger('= zip filter: %s',filter) + end local register, n = input.aux.register_file, 0 for i in z:files() do local path, name = i.filename:match(filter) @@ -5584,7 +5940,7 @@ else n = n + 1 end end - input.report('= zip entries',n) + input.logger('= zip entries: %s',n) return files end @@ -5601,6 +5957,8 @@ if not versions then versions = { } end -- special functions that deal with io +local format = string.format + if texconfig and not texlua then input.level = input.level or 0 @@ -5623,13 +5981,17 @@ if texconfig and not texlua then function input.show_load () end end - function input.finders.generic(instance,tag,filename,filetype) - local foundname = input.find_file(instance,filename,filetype) + function input.finders.generic(tag,filename,filetype) + local foundname = input.find_file(filename,filetype) if foundname and foundname ~= "" then - input.logger('+ ' .. tag .. ' finder',filename,'filetype') + if input.trace > 0 then + input.logger('+ finder: %s, file: %s', tag,filename) + end return foundname else - input.logger('- ' .. tag .. ' finder',filename,'filetype') + if input.trace > 0 then + input.logger('- finder: %s, file: %s', tag,filename) + end return unpack(input.finders.notfound) end end @@ -5642,7 +6004,9 @@ if texconfig and not texlua then local u = unicode.utftype(file_handle) local t = { } if u > 0 then - input.logger('+ ' .. tag .. ' opener (' .. unicode.utfname[u] .. ')',filename) + if input.trace > 0 then + input.logger('+ opener: %s (%s), file: %s',tag,unicode.utfname[u],filename) + end local l if u > 2 then l = unicode.utf32_to_utf8(file_handle:read("*a"),u==4) @@ -5657,8 +6021,11 @@ if texconfig and not texlua then handle = nil, noflines = #l, close = function() - input.logger('= ' .. tag .. ' closer (' .. unicode.utfname[u] .. ')',filename) + if input.trace > 0 then + input.logger('= closer: %s (%s), file: %s',tag,unicode.utfname[u],filename) + end input.show_close(filename) + t = nil end, --~ getline = function(n) --~ local line = t.lines[n] @@ -5693,7 +6060,9 @@ if texconfig and not texlua then end } else - input.logger('+ ' .. tag .. ' opener',filename) + if input.trace > 0 then + input.logger('+ opener: %s, file: %s',tag,filename) + end -- todo: file;name -> freeze / eerste regel scannen -> freeze local filters = input.filters t = { @@ -5713,9 +6082,12 @@ if texconfig and not texlua then return line end, close = function() - input.logger('= ' .. tag .. ' closer',filename) + if input.trace > 0 then + input.logger('= closer: %s, file: %s',tag,filename) + end input.show_close(filename) file_handle:close() + t = nil end, handle = function() return file_handle @@ -5729,7 +6101,7 @@ if texconfig and not texlua then return t end - function input.openers.generic(instance,tag,filename) + function input.openers.generic(tag,filename) if filename and filename ~= "" then local f = io.open(filename,"r") if f then @@ -5737,35 +6109,42 @@ if texconfig and not texlua then return input.openers.text_opener(filename,f,tag) end end - input.logger('- ' .. tag .. ' opener',filename) + if input.trace > 0 then + input.logger('- opener: %s, file: %s',tag,filename) + end return unpack(input.openers.notfound) end - function input.loaders.generic(instance,tag,filename) + function input.loaders.generic(tag,filename) if filename and filename ~= "" then local f = io.open(filename,"rb") if f then input.show_load(filename) - input.logger('+ ' .. tag .. ' loader',filename) + if input.trace > 0 then + input.logger('+ loader: %s, file: %s',tag,filename) + end local s = f:read("*a") + garbagecollector.check(s) f:close() if s then return true, s, #s end end end - input.logger('- ' .. tag .. ' loader',filename) + if input.trace > 0 then + input.logger('- loader: %s, file: %s',tag,filename) + end return unpack(input.loaders.notfound) end - function input.finders.tex(instance,filename,filetype) - return input.finders.generic(instance,'tex',filename,filetype) + function input.finders.tex(filename,filetype) + return input.finders.generic('tex',filename,filetype) end - function input.openers.tex(instance,filename) - return input.openers.generic(instance,'tex',filename) + function input.openers.tex(filename) + return input.openers.generic('tex',filename) end - function input.loaders.tex(instance,filename) - return input.loaders.generic(instance,'tex',filename) + function input.loaders.tex(filename) + return input.loaders.generic('tex',filename) end end @@ -5779,15 +6158,12 @@ if texconfig and not texlua then do ctx = ctx or { } - local ss = { } - - function ctx.writestatus(a,b) - local s = ss[a] - if not ss[a] then - s = a:rpadd(15) .. ": " - ss[a] = s + function ctx.writestatus(a,b,c,...) + if c then + texio.write_nl(("%-15s: %s\n"):format(a,b:format(c,...))) + else + texio.write_nl(("%-15s: %s\n"):format(a,b)) -- b can have %'s end - texio.write_nl(s .. b .. "\n") end -- this will become: ctx.install_statistics(fnc() return ..,.. end) etc @@ -5799,63 +6175,75 @@ if texconfig and not texlua then do if #tag > n then n = #tag end end + function ctx.memused() + -- collectgarbage("collect") + return string.format("%s MB (ctx: %s MB)",math.round(collectgarbage("count")), math.round(status.luastate_bytes/1000)) + end + function ctx.show_statistics() -- todo: move calls + local loadtime, register_statistics = input.loadtime, ctx.register_statistics if caches then - ctx.register_statistics("used config path", "%s", function() return caches.configpath(texmf.instance) end) - ctx.register_statistics("used cache path", "%s", function() return caches.path end) + register_statistics("used config path", "%s", function() return caches.configpath() end) + register_statistics("used cache path", "%s", function() return caches.temp() or "?" end) end if status.luabytecodes > 0 and input.storage and input.storage.done then - ctx.register_statistics("modules/dumps/instances", "%s/%s/%s", function() return status.luabytecodes-500, input.storage.done, status.luastates end) + register_statistics("modules/dumps/instances", "%s/%s/%s", function() return status.luabytecodes-500, input.storage.done, status.luastates end) end - if texmf.instance then - ctx.register_statistics("input load time", "%s seconds", function() return input.loadtime(texmf.instance) end) + if input.instance then + register_statistics("input load time", "%s seconds", function() return loadtime(input.instance) end) end if fonts then - ctx.register_statistics("fonts load time","%s seconds", function() return input.loadtime(fonts) end) + register_statistics("fonts load time","%s seconds", function() return loadtime(fonts) end) end if xml then - ctx.register_statistics("xml load time", "%s seconds, backreferences: %i, outer filtering time: %s", function() return input.loadtime(xml), #lxml.self, input.loadtime(lxml) end) + register_statistics("xml load time", "%s seconds, lpath calls: %s, cached calls: %s", function() + local stats = xml.statistics() + return loadtime(xml), stats.lpathcalls, stats.lpathcached + end) + register_statistics("lxml load time", "%s seconds preparation, backreferences: %i", function() + return loadtime(lxml), #lxml.self + end) end if mptopdf then - ctx.register_statistics("mps conversion time", "%s seconds", function() return input.loadtime(mptopdf) end) + register_statistics("mps conversion time", "%s seconds", function() return loadtime(mptopdf) end) end if nodes then - ctx.register_statistics("node processing time", "%s seconds (including kernel)", function() return input.loadtime(nodes) end) + register_statistics("node processing time", "%s seconds including kernel", function() return loadtime(nodes) end) end if kernel then - ctx.register_statistics("kernel processing time", "%s seconds", function() return input.loadtime(kernel) end) + register_statistics("kernel processing time", "%s seconds", function() return loadtime(kernel) end) end if attributes then - ctx.register_statistics("attribute processing time", "%s seconds", function() return input.loadtime(attributes) end) + register_statistics("attribute processing time", "%s seconds", function() return loadtime(attributes) end) end if languages then - ctx.register_statistics("language load time", "%s seconds, n=%s", function() return input.loadtime(languages), languages.hyphenation.n() end) + register_statistics("language load time", "%s seconds, n=%s", function() return loadtime(languages), languages.hyphenation.n() end) end if figures then - ctx.register_statistics("graphics processing time", "%s seconds, n=%s (including tex)", function() return input.loadtime(figures), figures.n or "?" end) + register_statistics("graphics processing time", "%s seconds including tex, n=%s", function() return loadtime(figures), figures.n or "?" end) end if metapost then - ctx.register_statistics("metapost processing time", "%s seconds, loading: %s seconds, execution: %s seconds, n: %s", function() return input.loadtime(metapost), input.loadtime(mplib), input.loadtime(metapost.exectime), metapost.n end) + register_statistics("metapost processing time", "%s seconds, loading: %s seconds, execution: %s seconds, n: %s", function() return loadtime(metapost), loadtime(mplib), loadtime(metapost.exectime), metapost.n end) end - if status.luastate_bytes then - ctx.register_statistics("current memory usage", "%s bytes", function() return status.luastate_bytes end) + if status.luastate_bytes and ctx.memused then + register_statistics("current memory usage", "%s", ctx.memused) end if nodes then - ctx.register_statistics("cleaned up reserved nodes", "%s nodes, %s lists of %s", function() return nodes.cleanup_reserved(tex.count[24]) end) -- \topofboxstack + register_statistics("cleaned up reserved nodes", "%s nodes, %s lists of %s", function() return nodes.cleanup_reserved(tex.count[24]) end) -- \topofboxstack end if status.node_mem_usage then - ctx.register_statistics("node memory usage", "%s", function() return status.node_mem_usage end) + register_statistics("node memory usage", "%s", function() return status.node_mem_usage end) end if languages then - ctx.register_statistics("loaded patterns", "%s", function() return languages.logger.report() end) + register_statistics("loaded patterns", "%s", function() return languages.logger.report() end) end if fonts then - ctx.register_statistics("loaded fonts", "%s", function() return fonts.logger.report() end) + register_statistics("loaded fonts", "%s", function() return fonts.logger.report() end) end if xml then -- so we are in mkiv, we need a different check - ctx.register_statistics("runtime", "%s seconds, %i processed pages, %i shipped pages, %.3f pages/second", function() - input.stoptiming(texmf) - local runtime = input.loadtime(texmf) + register_statistics("runtime", "%s seconds, %i processed pages, %i shipped pages, %.3f pages/second", function() + input.stoptiming(input.instance) + local runtime = loadtime(input.instance) local shipped = tex.count['nofshipouts'] local pages = tex.count['realpageno'] - 1 local persecond = shipped / runtime @@ -5864,8 +6252,8 @@ if texconfig and not texlua then do end for _, t in ipairs(statusinfo) do local tag, pattern, fnc = t[1], t[2], t[3] - ctx.writestatus("mkiv lua stats", string.format("%s - %s", tag:rpadd(n," "), pattern:format(fnc()))) - end + ctx.writestatus("mkiv lua stats", "%s - %s", tag:rpadd(n," "), pattern:format(fnc())) + end-- input.expanded_path_list("osfontdir") end end end @@ -5878,65 +6266,65 @@ if texconfig and not texlua then -- if still present, we overload kpse (put it off-line so to say) - if not texmf then texmf = { } end + input.starttiming(input.instance) - input.starttiming(texmf) + if not input.instance then - if not texmf.instance then + if not input.instance then -- prevent a second loading - if not texmf.instance then -- prevent a second loading + input.instance = input.reset() + input.instance.progname = 'context' + input.instance.engine = 'luatex' + input.instance.validfile = input.validctxfile - texmf.instance = input.reset() - texmf.instance.progname = environment.progname or 'context' - texmf.instance.engine = environment.engine or 'luatex' - texmf.instance.validfile = input.validctxfile - - input.load(texmf.instance) + input.load() end if callback then - callback.register('find_read_file' , function(id,name) return input.findtexfile(texmf.instance,name) end) - callback.register('open_read_file' , function( name) return input.opentexfile(texmf.instance,name) end) + callback.register('find_read_file' , function(id,name) return input.findtexfile(name) end) + callback.register('open_read_file' , function( name) return input.opentexfile(name) end) end if callback then - callback.register('find_data_file' , function(name) return input.findbinfile(texmf.instance,name,"tex") end) - callback.register('find_enc_file' , function(name) return input.findbinfile(texmf.instance,name,"enc") end) - callback.register('find_font_file' , function(name) return input.findbinfile(texmf.instance,name,"tfm") end) - callback.register('find_format_file' , function(name) return input.findbinfile(texmf.instance,name,"fmt") end) - callback.register('find_image_file' , function(name) return input.findbinfile(texmf.instance,name,"tex") end) - callback.register('find_map_file' , function(name) return input.findbinfile(texmf.instance,name,"map") end) - callback.register('find_ocp_file' , function(name) return input.findbinfile(texmf.instance,name,"ocp") end) - callback.register('find_opentype_file' , function(name) return input.findbinfile(texmf.instance,name,"otf") end) - callback.register('find_output_file' , function(name) return name end) - callback.register('find_pk_file' , function(name) return input.findbinfile(texmf.instance,name,"pk") end) - callback.register('find_sfd_file' , function(name) return input.findbinfile(texmf.instance,name,"sfd") end) - callback.register('find_truetype_file' , function(name) return input.findbinfile(texmf.instance,name,"ttf") end) - callback.register('find_type1_file' , function(name) return input.findbinfile(texmf.instance,name,"pfb") end) - callback.register('find_vf_file' , function(name) return input.findbinfile(texmf.instance,name,"vf") end) - - callback.register('read_data_file' , function(file) return input.loadbinfile(texmf.instance,file,"tex") end) - callback.register('read_enc_file' , function(file) return input.loadbinfile(texmf.instance,file,"enc") end) - callback.register('read_font_file' , function(file) return input.loadbinfile(texmf.instance,file,"tfm") end) + callback.register('find_data_file' , function(name) return input.findbinfile(name,"tex") end) + callback.register('find_enc_file' , function(name) return input.findbinfile(name,"enc") end) + callback.register('find_font_file' , function(name) return input.findbinfile(name,"tfm") end) + callback.register('find_format_file' , function(name) return input.findbinfile(name,"fmt") end) + callback.register('find_image_file' , function(name) return input.findbinfile(name,"tex") end) + callback.register('find_map_file' , function(name) return input.findbinfile(name,"map") end) + callback.register('find_ocp_file' , function(name) return input.findbinfile(name,"ocp") end) + callback.register('find_opentype_file' , function(name) return input.findbinfile(name,"otf") end) + callback.register('find_output_file' , function(name) return name end) + callback.register('find_pk_file' , function(name) return input.findbinfile(name,"pk") end) + callback.register('find_sfd_file' , function(name) return input.findbinfile(name,"sfd") end) + callback.register('find_truetype_file' , function(name) return input.findbinfile(name,"ttf") end) + callback.register('find_type1_file' , function(name) return input.findbinfile(name,"pfb") end) + callback.register('find_vf_file' , function(name) return input.findbinfile(name,"vf") end) + + callback.register('read_data_file' , function(file) return input.loadbinfile(file,"tex") end) + callback.register('read_enc_file' , function(file) return input.loadbinfile(file,"enc") end) + callback.register('read_font_file' , function(file) return input.loadbinfile(file,"tfm") end) -- format -- image - callback.register('read_map_file' , function(file) return input.loadbinfile(texmf.instance,file,"map") end) - callback.register('read_ocp_file' , function(file) return input.loadbinfile(texmf.instance,file,"ocp") end) - callback.register('read_opentype_file' , function(file) return input.loadbinfile(texmf.instance,file,"otf") end) + callback.register('read_map_file' , function(file) return input.loadbinfile(file,"map") end) + callback.register('read_ocp_file' , function(file) return input.loadbinfile(file,"ocp") end) + callback.register('read_opentype_file' , function(file) return input.loadbinfile(file,"otf") end) -- output - callback.register('read_pk_file' , function(file) return input.loadbinfile(texmf.instance,file,"pk") end) - callback.register('read_sfd_file' , function(file) return input.loadbinfile(texmf.instance,file,"sfd") end) - callback.register('read_truetype_file' , function(file) return input.loadbinfile(texmf.instance,file,"ttf") end) - callback.register('read_type1_file' , function(file) return input.loadbinfile(texmf.instance,file,"pfb") end) - callback.register('read_vf_file' , function(file) return input.loadbinfile(texmf.instance,file,"vf" ) end) + callback.register('read_pk_file' , function(file) return input.loadbinfile(file,"pk") end) + callback.register('read_sfd_file' , function(file) return input.loadbinfile(file,"sfd") end) + callback.register('read_truetype_file' , function(file) return input.loadbinfile(file,"ttf") end) + callback.register('read_type1_file' , function(file) return input.loadbinfile(file,"pfb") end) + callback.register('read_vf_file' , function(file) return input.loadbinfile(file,"vf" ) end) end - if callback and environment.aleph_mode then - callback.register('find_font_file' , function(name) return input.findbinfile(texmf.instance,name,"ofm") end) - callback.register('read_font_file' , function(file) return input.loadbinfile(texmf.instance,file,"ofm") end) - callback.register('find_vf_file' , function(name) return input.findbinfile(texmf.instance,name,"ovf") end) - callback.register('read_vf_file' , function(file) return input.loadbinfile(texmf.instance,file,"ovf") end) + if input.aleph_mode == nil then environment.aleph_mode = true end -- some day we will drop omega font support + + if callback and input.aleph_mode then + callback.register('find_font_file' , function(name) return input.findbinfile(name,"ofm") end) + callback.register('read_font_file' , function(file) return input.loadbinfile(file,"ofm") end) + callback.register('find_vf_file' , function(name) return input.findbinfile(name,"ovf") end) + callback.register('read_vf_file' , function(file) return input.loadbinfile(file,"ovf") end) end if callback then @@ -6024,16 +6412,16 @@ if texconfig and not texlua then if kpse then function kpse.find_file(filename,filetype,mustexist) - return input.find_file(texmf.instance,filename,filetype,mustexist) + return input.find_file(filename,filetype,mustexist) end function kpse.expand_path(variable) - return input.expand_path(texmf.instance,variable) + return input.expand_path(variable) end function kpse.expand_var(variable) - return input.expand_var(texmf.instance,variable) + return input.expand_var(variable) end function kpse.expand_braces(variable) - return input.expand_braces(texmf.instance,variable) + return input.expand_braces(variable) end end @@ -6060,7 +6448,7 @@ if texconfig and not texlua then function luatex.variables() local t, x = { }, nil for _,v in pairs(luatex.variablenames) do - x = input.var_value(texmf.instance,v) + x = input.var_value(v) if x and x:find("^%d+$") then t[v] = tonumber(x) end @@ -6084,28 +6472,34 @@ if texconfig and not texlua then end --- some tex basics +-- some tex basics, maybe this will move to ctx -if not cs then cs = { } end +if tex then -function cs.def(k,v) - tex.sprint(tex.texcatcodes, "\\def\\" .. k .. "{" .. v .. "}") -end + local texsprint, texwrite = tex.sprint, tex.write -function cs.chardef(k,v) - tex.sprint(tex.texcatcodes, "\\chardef\\" .. k .. "=" .. v .. "\\relax") -end + if not cs then cs = { } end -function cs.boolcase(b) - if b then tex.write(1) else tex.write(0) end -end + function cs.def(k,v) + texsprint(tex.texcatcodes, "\\def\\" .. k .. "{" .. v .. "}") + end -function cs.testcase(b) - if b then - tex.sprint(tex.texcatcodes, "\\firstoftwoarguments") - else - tex.sprint(tex.texcatcodes, "\\secondoftwoarguments") + function cs.chardef(k,v) + texsprint(tex.texcatcodes, "\\chardef\\" .. k .. "=" .. v .. "\\relax") + end + + function cs.boolcase(b) + if b then texwrite(1) else texwrite(0) end + end + + function cs.testcase(b) + if b then + texsprint(tex.texcatcodes, "\\firstoftwoarguments") + else + texsprint(tex.texcatcodes, "\\secondoftwoarguments") + end end + end @@ -6235,9 +6629,11 @@ own.libs = { -- todo: check which ones a 'l-dir.lua', 'l-boolean.lua', 'l-unicode.lua', + 'l-math.lua', 'l-utils.lua', 'luat-lib.lua', 'luat-inp.lua', + 'luat-log.lua', 'luat-tmp.lua', 'luat-zip.lua', 'luat-tex.lua', @@ -6291,19 +6687,21 @@ if not input then os.exit() end -instance = input.reset() +input.instance = input.reset() input.verbose = environment.arguments["verbose"] or false -input.banner = 'LuaTools | ' +input.banner = 'LuaTools' utils.report = input.report input.defaultlibs = { -- not all are needed 'l-string.lua', 'l-lpeg.lua', 'l-table.lua', 'l-boolean.lua', 'l-number.lua', 'l-set.lua', 'l-unicode.lua', - 'l-md5.lua', 'l-os.lua', 'l-io.lua', 'l-file.lua', 'l-url.lua', 'l-dir.lua', 'l-utils.lua', 'l-tex.lua', - 'luat-env.lua', 'luat-lib.lua', 'luat-inp.lua', 'luat-tmp.lua', 'luat-zip.lua', 'luat-tex.lua' + 'l-md5.lua', 'l-os.lua', 'l-io.lua', 'l-file.lua', 'l-url.lua', 'l-dir.lua', 'l-utils.lua', 'l-dimen.lua', + 'luat-lib.lua', 'luat-inp.lua', 'luat-env.lua', 'luat-tmp.lua', 'luat-zip.lua', 'luat-tex.lua' } -- todo: use environment.argument() instead of environment.arguments[] +local instance = input.instance + instance.engine = environment.arguments["engine"] or 'luatex' instance.progname = environment.arguments["progname"] or 'context' instance.luaname = environment.arguments["luafile"] or "" -- environment.ownname or "" @@ -6328,19 +6726,19 @@ if environment.arguments["minimize"] the end end -function input.my_prepare_a(instance) - input.resetconfig(instance) - input.identify_cnf(instance) - input.load_lua(instance) - input.expand_variables(instance) - input.load_cnf(instance) - input.expand_variables(instance) +function input.my_prepare_a() + input.resetconfig() + input.identify_cnf() + input.load_lua() + input.expand_variables() + input.load_cnf() + input.expand_variables() end -function input.my_prepare_b(instance) - input.my_prepare_a(instance) - input.load_hash(instance) - input.automount(instance) +function input.my_prepare_b() + input.my_prepare_a() + input.load_hash() + input.automount() end -- barename @@ -6382,10 +6780,11 @@ messages.help = [[ --lsr use lsr and cnf directly ]] -function input.my_make_format(instance,texname) +function input.my_make_format(texname) + local instance = input.instance if texname and texname ~= "" then if input.usecache then - local path = file.join(caches.setpath(instance,"formats")) -- maybe platform + local path = file.join(caches.setpath("formats")) -- maybe platform if path and lfs then lfs.chdir(path) end @@ -6394,22 +6793,22 @@ function input.my_make_format(instance,t if barename == texname then texname = texname .. ".tex" end - local fullname = input.find_files(instance,texname)[1] or "" + local fullname = input.find_files(texname)[1] or "" if fullname == "" then - input.report("no tex file with name",texname) + input.report("no tex file with name: %s",texname) else local luaname, lucname, luapath, lualibs = "", "", "", { } -- the following is optional, since context.lua can also -- handle this collect and compile business if environment.arguments["compile"] then if luaname == "" then luaname = barename end - input.report("creating initialization file " .. luaname) + input.report("creating initialization file: %s",luaname) luapath = file.dirname(luaname) if luapath == "" then luapath = file.dirname(texname) end if luapath == "" then - luapath = file.dirname(input.find_files(instance,texname)[1] or "") + luapath = file.dirname(input.find_files(texname)[1] or "") end lualibs = string.split(instance.lualibs,",") luaname = file.basename(barename .. ".lua") @@ -6419,27 +6818,28 @@ function input.my_make_format(instance,t if lualibs[1] then local firstlib = file.join(luapath,lualibs[1]) if not lfs.isfile(firstlib) then - local foundname = input.find_files(instance,lualibs[1])[1] + local foundname = input.find_files(lualibs[1])[1] if foundname then - input.report("located library path : " .. luapath) + input.report("located library path: %s",luapath) luapath = file.dirname(foundname) end end end - input.report("using library path : " .. luapath) - input.report("using lua libraries: " .. table.join(lualibs," ")) + input.report("using library path: %s",luapath) + input.report("using lua libraries: %s",table.join(lualibs," ")) utils.merger.selfcreate(lualibs,luapath,luaname) - if utils.lua.compile(luaname, lucname) and io.exists(lucname) then + local strip = input.boolean_variable("LUACSTRIP", true) + if utils.lua.compile(luaname,lucname,false,strip) and io.exists(lucname) then luaname = lucname - input.report("using compiled initialization file " .. lucname) + input.report("using compiled initialization file: %s",lucname) else - input.report("using uncompiled initialization file " .. luaname) + input.report("using uncompiled initialization file: %s",luaname) end else for _, v in pairs({instance.luaname, instance.progname, barename}) do v = string.gsub(v..".lua","%.lua%.lua$",".lua") if v and (v ~= "") then - luaname = input.find_files(instance,v)[1] or "" + luaname = input.find_files(v)[1] or "" if luaname ~= "" then break end @@ -6448,22 +6848,30 @@ function input.my_make_format(instance,t end if luaname == "" then input.reportlines(messages.no_ini_file) - input.report("texname : " .. texname) - input.report("luaname : " .. instance.luaname) - input.report("progname : " .. instance.progname) - input.report("barename : " .. barename) - else - input.report("using lua initialization file " .. luaname) + input.report("texname : %s",texname) + input.report("luaname : %s",instance.luaname) + input.report("progname: %s",instance.progname) + input.report("barename: %s",barename) + else + input.report("using lua initialization file: %s",luaname) + local mp = dir.glob(file.stripsuffix(file.basename(luaname)).."-*.mem") + if mp and #mp > 0 then + for _, name in ipairs(mp) do + input.report("removing related mplib format %s", file.basename(name)) + os.remove(name) + end + end local flags = { "--ini" } if environment.arguments["mkii"] then flags[#flags+1] = "--progname=" .. instance.progname else flags[#flags+1] = "--lua=" .. string.quote(luaname) end - local bs = (environment.platform == "unix" and "\\\\") or "\\" -- todo: make a function + local bs = (os.platform == "unix" and "\\\\") or "\\" -- todo: make a function local command = "luatex ".. table.concat(flags," ") .. " " .. string.quote(fullname) .. " " .. bs .. "dump" - input.report("running command: " .. command .. "\n") + input.report("running command: %s\n",command) os.spawn(command) + -- todo: do a dummy run that generates the related metafun and mfplain formats end end else @@ -6471,22 +6879,22 @@ function input.my_make_format(instance,t end end -function input.my_run_format(instance,name,data,more) +function input.my_run_format(name,data,more) -- hm, rather old code here; we can now use the file.whatever functions if name and (name ~= "") then local barename = name:gsub("%.%a+$","") local fmtname = "" if input.usecache then - local path = file.join(caches.setpath(instance,"formats")) -- maybe platform + local path = file.join(caches.setpath("formats")) -- maybe platform fmtname = file.join(path,barename..".fmt") or "" end if fmtname == "" then - fmtname = input.find_files(instance,barename..".fmt")[1] or "" + fmtname = input.find_files(barename..".fmt")[1] or "" end fmtname = input.clean_path(fmtname) barename = fmtname:gsub("%.%a+$","") if fmtname == "" then - input.report("no format with name",name) + input.report("no format with name: %s",name) else local luaname = barename .. ".luc" local f = io.open(luaname) @@ -6497,11 +6905,11 @@ function input.my_run_format(instance,na if f then f:close() local command = "luatex --fmt=" .. string.quote(barename) .. " --lua=" .. string.quote(luaname) .. " " .. string.quote(data) .. " " .. string.quote(more) - input.report("running command: " .. command) + input.report("running command: %s",command) os.spawn(command) else - input.report("using format name",fmtname) - input.report("no luc/lua with name",barename) + input.report("using format name: %s",fmtname) + input.report("no luc/lua with name: %s",barename) end end end @@ -6519,8 +6927,9 @@ local function tabstr(str) end end -local function list(instance,list) - local pat = string.upper(instance.pattern or "","") +local function list(list) + local instance = input.instance + local pat = string.upper(pattern or "","") for _,key in pairs(table.sortedkeys(list)) do if instance.pattern == "" or string.find(key:upper(),pat) then if instance.kpseonly then @@ -6534,10 +6943,11 @@ local function list(instance,list) end end -function input.listers.variables (instance) list(instance,instance.variables ) end -function input.listers.expansions(instance) list(instance,instance.expansions) end +function input.listers.variables () list(input.instance.variables ) end +function input.listers.expansions() list(input.instance.expansions) end -function input.listers.configurations(instance) +function input.listers.configurations() + local instance = input.instance for _,key in pairs(table.sortedkeys(instance.kpsevars)) do if not instance.pattern or (instance.pattern=="") or key:find(instance.pattern) then print(key.."\n") @@ -6552,61 +6962,58 @@ function input.listers.configurations(in end end -input.report(banner,"\n") +input.report("%s\n",banner) local ok = true if environment.arguments["find-file"] then - input.my_prepare_b(instance) + input.my_prepare_b() instance.format = environment.arguments["format"] or instance.format if instance.pattern then instance.allresults = true - input.for_files(instance, input.find_files, { instance.pattern }, instance.my_format) + input.for_files(input.find_files, { instance.pattern }, instance.my_format) else - input.for_files(instance, input.find_files, environment.files, instance.my_format) + input.for_files(input.find_files, environment.files, instance.my_format) end elseif environment.arguments["find-path"] then - input.my_prepare_b(instance) - local path = input.find_file(instance, environment.files[1], instance.my_format) + input.my_prepare_b() + local path = input.find_file(environment.files[1], instance.my_format) if input.verbose then input.report(file.dirname(path)) else print(file.dirname(path)) end ---~ elseif environment.arguments["first-writable-path"] then ---~ input.my_prepare_b(instance) ---~ input.report(input.first_writable_path(instance,environment.files[1] or ".")) elseif environment.arguments["run"] then - input.my_prepare_a(instance) -- ! no need for loading databases + input.my_prepare_a() -- ! no need for loading databases input.verbose = true - input.my_run_format(instance,environment.files[1] or "",environment.files[2] or "",environment.files[3] or "") + input.my_run_format(environment.files[1] or "",environment.files[2] or "",environment.files[3] or "") elseif environment.arguments["fmt"] then - input.my_prepare_a(instance) -- ! no need for loading databases + input.my_prepare_a() -- ! no need for loading databases input.verbose = true - input.my_run_format(instance,environment.arguments["fmt"], environment.files[1] or "",environment.files[2] or "") + input.my_run_format(environment.arguments["fmt"], environment.files[1] or "",environment.files[2] or "") elseif environment.arguments["expand-braces"] then - input.my_prepare_a(instance) - input.for_files(instance, input.expand_braces, environment.files) + input.my_prepare_a() + input.for_files(input.expand_braces, environment.files) elseif environment.arguments["expand-path"] then - input.my_prepare_a(instance) - input.for_files(instance, input.expand_path, environment.files) + input.my_prepare_a() + input.for_files(input.expand_path, environment.files) elseif environment.arguments["expand-var"] or environment.arguments["expand-variable"] then - input.my_prepare_a(instance) - input.for_files(instance, input.expand_var, environment.files) + input.my_prepare_a() + input.for_files(input.expand_var, environment.files) elseif environment.arguments["show-path"] or environment.arguments["path-value"] then - input.my_prepare_a(instance) - input.for_files(instance, input.show_path, environment.files) + input.my_prepare_a() + input.for_files(input.show_path, environment.files) elseif environment.arguments["var-value"] or environment.arguments["show-value"] then - input.my_prepare_a(instance) - input.for_files(instance, input.var_value, environment.files) + input.my_prepare_a() + input.for_files(input.var_value, environment.files) elseif environment.arguments["format-path"] then - input.my_prepare_b(instance) - input.report(caches.setpath(instance,"format")) + input.my_prepare_b() + input.report(caches.setpath("format")) elseif instance.pattern then -- brrr - input.my_prepare_b(instance) + input.my_prepare_b() instance.format = environment.arguments["format"] or instance.format instance.allresults = true - input.for_files(instance, input.find_files, { instance.pattern }, instance.my_format) + input.for_files(input.find_files, { instance.pattern }, instance.my_format) elseif environment.arguments["generate"] then instance.renewcache = true input.verbose = true @@ -6614,46 +7021,36 @@ elseif environment.arguments["generate"] elseif environment.arguments["make"] or environment.arguments["ini"] or environment.arguments["compile"] then input.my_prepare_b(instance) input.verbose = true - input.my_make_format(instance,environment.files[1] or "") + input.my_make_format(environment.files[1] or "") elseif environment.arguments["selfmerge"] then utils.merger.selfmerge(own.name,own.libs,own.list) elseif environment.arguments["selfclean"] then utils.merger.selfclean(own.name) elseif environment.arguments["selfupdate"] then - input.my_prepare_b(instance) + input.my_prepare_b() input.verbose = true - input.update_script(instance,own.name,"luatools") + input.update_script(own.name,"luatools") elseif environment.arguments["variables"] or environment.arguments["show-variables"] then - input.my_prepare_a(instance) - input.listers.variables(instance) + input.my_prepare_a() + input.listers.variables() elseif environment.arguments["expansions"] or environment.arguments["show-expansions"] then - input.my_prepare_a(instance) - input.listers.expansions(instance) + input.my_prepare_a() + input.listers.expansions() elseif environment.arguments["configurations"] or environment.arguments["show-configurations"] then - input.my_prepare_a(instance) - input.listers.configurations(instance) + input.my_prepare_a() + input.listers.configurations() elseif environment.arguments["help"] or (environment.files[1]=='help') or (#environment.files==0) then - if not input.verbose then - input.verbose = true - input.report(banner,"\n") - end - input.reportlines(messages.help) + input.help(banner,messages.help) else - input.my_prepare_b(instance) - input.for_files(instance, input.find_files, environment.files, instance.my_format) + input.my_prepare_b() + input.for_files(input.find_files, environment.files, instance.my_format) end if input.verbose then input.report("") - input.report(string.format("runtime: %0.3f seconds",os.runtime())) + input.report("runtime: %0.3f seconds",os.runtime()) end ---~ if ok then ---~ input.report("exit code: 0") os.exit(0) ---~ else ---~ input.report("exit code: 1") os.exit(1) ---~ end - -if environment.platform == "unix" then +if os.platform == "unix" then io.write("\n") end --- texmf-dist/scripts/context/lua/mtxrun.lua +++ texmf-dist/scripts/context/lua/mtxrun.lua 2008-10-30 14:05:20.903589570 +0100 @@ -37,7 +37,7 @@ if not modules then modules = { } end mo -- remember for subruns: _CTX_K_S_#{original}_ -- remember for subruns: TEXMFSTART.#{original} [tex.rb texmfstart.rb] -banner = "version 1.0.2 - 2007+ - PRAGMA ADE / CONTEXT" +banner = "version 1.1.2 - 2007+ - PRAGMA ADE / CONTEXT" -- not local texlua = true -- begin library merge @@ -125,7 +125,7 @@ end function string:splitchr(chr) if #self > 0 then local t = { } - for s in string.gmatch(self..chr,"(.-)"..chr) do + for s in (self..chr):gmatch("(.-)"..chr) do t[#t+1] = s end return t @@ -134,22 +134,6 @@ function string:splitchr(chr) end end ---~ function string.piecewise(str, pat, fnc) -- variant of split ---~ local fpat = "(.-)"..pat ---~ local last_end = 1 ---~ local s, e, cap = string.find(str, fpat, 1) ---~ while s ~= nil do ---~ if s~=1 or cap~="" then ---~ fnc(cap) ---~ end ---~ last_end = e+1 ---~ s, e, cap = string.find(str, fpat, last_end) ---~ end ---~ if last_end <= #str then ---~ fnc((string.sub(str,last_end))) ---~ end ---~ end - function string.piecewise(str, pat, fnc) -- variant of split for k in string.splitter(str,pat) do fnc(k) end end @@ -191,7 +175,7 @@ end --~ end end -string.chr_to_esc = { +local chr_to_esc = { ["%"] = "%%", ["."] = "%.", ["+"] = "%+", ["-"] = "%-", ["*"] = "%*", @@ -201,16 +185,18 @@ string.chr_to_esc = { ["{"] = "%{", ["}"] = "%}" } +string.chr_to_esc = chr_to_esc + function string:esc() -- variant 2 - return (self:gsub("(.)",string.chr_to_esc)) + return (self:gsub("(.)",chr_to_esc)) end -function string.unquote(str) - return (str:gsub("^([\"\'])(.*)%1$","%2")) +function string:unquote() + return (self:gsub("^([\"\'])(.*)%1$","%2")) end -function string.quote(str) - return '"' .. str:unquote() .. '"' +function string:quote() + return '"' .. self:unquote() .. '"' end function string:count(pattern) -- variant 3 @@ -460,6 +446,30 @@ function string:splitlines() return capture:match(self) end +--~ local p = lpeg.splitat("->",false) print(p:match("oeps->what->more")) -- oeps what more +--~ local p = lpeg.splitat("->",true) print(p:match("oeps->what->more")) -- oeps what->more +--~ local p = lpeg.splitat("->",false) print(p:match("oeps")) -- oeps +--~ local p = lpeg.splitat("->",true) print(p:match("oeps")) -- oeps + +local splitters_s, splitters_m = { }, { } + +function lpeg.splitat(separator,single) + local splitter = (single and splitters_s[separator]) or splitters_m[separator] + if not splitter then + separator = lpeg.P(separator) + if single then + local other, any = lpeg.C((1 - separator)^0), lpeg.P(1) + splitter = other * (separator * lpeg.C(any^0) + "") + splitters_s[separator] = splitter + else + local other = lpeg.C((1 - separator)^0) + splitter = other * (separator * other)^0 + splitters_m[separator] = splitter + end + end + return splitter +end + -- filename : l-table.lua -- comment : split off from luat-lib @@ -471,11 +481,15 @@ if not versions then versions = { } end table.join = table.concat +local concat, sort, insert, remove = table.concat, table.sort, table.insert, table.remove +local format = string.format +local getmetatable, setmetatable = getmetatable, setmetatable +local pairs, ipairs, type, next, tostring = pairs, ipairs, type, next, tostring + function table.strip(tab) local lst = { } - for k, v in ipairs(tab) do - -- s = string.gsub(v, "^%s*(.-)%s*$", "%1") - s = v:gsub("^%s*(.-)%s*$", "%1") + for i=1,#tab do + local s = tab[i]:gsub("^%s*(.-)%s*$","%1") if s == "" then -- skip this one else @@ -485,16 +499,7 @@ function table.strip(tab) return lst end ---~ function table.sortedkeys(tab) ---~ local srt = { } ---~ for key,_ in pairs(tab) do ---~ srt[#srt+1] = key ---~ end ---~ table.sort(srt) ---~ return srt ---~ end - -function table.sortedkeys(tab) +local function sortedkeys(tab) local srt, kind = { }, 0 -- 0=unknown 1=string, 2=number 3=mixed for key,_ in pairs(tab) do srt[#srt+1] = key @@ -514,22 +519,34 @@ function table.sortedkeys(tab) end end if kind == 0 or kind == 3 then - table.sort(srt,function(a,b) return (tostring(a) < tostring(b)) end) + sort(srt,function(a,b) return (tostring(a) < tostring(b)) end) else - table.sort(srt) + sort(srt) end return srt end +local function sortedhashkeys(tab) -- fast one + local srt = { } + for key,_ in pairs(tab) do + srt[#srt+1] = key + end + sort(srt) + return srt +end + +table.sortedkeys = sortedkeys +table.sortedhashkeys = sortedhashkeys + function table.append(t, list) for _,v in pairs(list) do - table.insert(t,v) + insert(t,v) end end function table.prepend(t, list) for k,v in pairs(list) do - table.insert(t,k,v) + insert(t,k,v) end end @@ -576,70 +593,57 @@ function table.imerged(...) return tmp end -if not table.fastcopy then do - - local type, pairs, getmetatable, setmetatable = type, pairs, getmetatable, setmetatable - - local function fastcopy(old) -- fast one - if old then - local new = { } - for k,v in pairs(old) do - if type(v) == "table" then - new[k] = fastcopy(v) -- was just table.copy - else - new[k] = v - end - end - local mt = getmetatable(old) - if mt then - setmetatable(new,mt) +local function fastcopy(old) -- fast one + if old then + local new = { } + for k,v in pairs(old) do + if type(v) == "table" then + new[k] = fastcopy(v) -- was just table.copy + else + new[k] = v end - return new - else - return { } end + local mt = getmetatable(old) + if mt then + setmetatable(new,mt) + end + return new + else + return { } end +end - table.fastcopy = fastcopy - -end end - -if not table.copy then do - - local type, pairs, getmetatable, setmetatable = type, pairs, getmetatable, setmetatable - - local function copy(t, tables) -- taken from lua wiki, slightly adapted - tables = tables or { } - local tcopy = {} - if not tables[t] then - tables[t] = tcopy - end - for i,v in pairs(t) do -- brrr, what happens with sparse indexed - if type(i) == "table" then - if tables[i] then - i = tables[i] - else - i = copy(i, tables) - end - end - if type(v) ~= "table" then - tcopy[i] = v - elseif tables[v] then - tcopy[i] = tables[v] +local function copy(t, tables) -- taken from lua wiki, slightly adapted + tables = tables or { } + local tcopy = {} + if not tables[t] then + tables[t] = tcopy + end + for i,v in pairs(t) do -- brrr, what happens with sparse indexed + if type(i) == "table" then + if tables[i] then + i = tables[i] else - tcopy[i] = copy(v, tables) + i = copy(i, tables) end end - local mt = getmetatable(t) - if mt then - setmetatable(tcopy,mt) + if type(v) ~= "table" then + tcopy[i] = v + elseif tables[v] then + tcopy[i] = tables[v] + else + tcopy[i] = copy(v, tables) end - return tcopy end + local mt = getmetatable(t) + if mt then + setmetatable(tcopy,mt) + end + return tcopy +end - table.copy = copy - -end end +table.fastcopy = fastcopy +table.copy = copy -- rougly: copy-loop : unpack : sub == 0.9 : 0.4 : 0.45 (so in critical apps, use unpack) @@ -668,334 +672,363 @@ function table.starts_at(t) return ipairs(t,1)(t,0) end -do +function table.tohash(t,value) + local h = { } + if value == nil then value = true end + for _, v in pairs(t) do -- no ipairs here + h[v] = value + end + return h +end + +function table.fromhash(t) + local h = { } + for k, v in pairs(t) do -- no ipairs here + if v then h[#h+1] = k end + end + return h +end - -- one of my first exercises in lua ... +--~ print(table.serialize(t), "\n") +--~ print(table.serialize(t,"name"), "\n") +--~ print(table.serialize(t,false), "\n") +--~ print(table.serialize(t,true), "\n") +--~ print(table.serialize(t,"name",true), "\n") +--~ print(table.serialize(t,"name",true,true), "\n") - -- 34.055.092 32.403.326 arabtype.tma - -- 1.620.614 1.513.863 lmroman10-italic.tma - -- 1.325.585 1.233.044 lmroman10-regular.tma - -- 1.248.157 1.158.903 lmsans10-regular.tma - -- 194.646 153.120 lmtypewriter10-regular.tma - -- 1.771.678 1.658.461 palatinosanscom-bold.tma - -- 1.695.251 1.584.491 palatinosanscom-regular.tma - -- 13.736.534 13.409.446 zapfinoextraltpro.tma - - -- 13.679.038 11.774.106 arabtype.tmc - -- 886.248 754.944 lmroman10-italic.tmc - -- 729.828 466.864 lmroman10-regular.tmc - -- 688.482 441.962 lmsans10-regular.tmc - -- 128.685 95.853 lmtypewriter10-regular.tmc - -- 715.929 582.985 palatinosanscom-bold.tmc - -- 669.942 540.126 palatinosanscom-regular.tmc - -- 1.560.588 1.317.000 zapfinoextraltpro.tmc - - table.serialize_functions = true - table.serialize_compact = true - table.serialize_inline = true +table.serialize_functions = true +table.serialize_compact = true +table.serialize_inline = true + +local noquotes, hexify, handle, reduce, compact, inline, functions + +local reserved = table.tohash { -- intercept a language flaw, no reserved words as key + 'and', 'break', 'do', 'else', 'elseif', 'end', 'false', 'for', 'function', 'if', + 'in', 'local', 'nil', 'not', 'or', 'repeat', 'return', 'then', 'true', 'until', 'while', +} - local function key(k) - if type(k) == "number" then -- or k:find("^%d+$") then - return "["..k.."]" - elseif noquotes and k:find("^%a[%a%d%_]*$") then - return k +local function key(k) + if type(k) == "number" then -- or k:find("^%d+$") then + if hexify then + return ("[0x%04X]"):format(k) else - return '["'..k..'"]' + return "["..k.."]" end + elseif noquotes and not reserved[k] and k:find("^%a[%a%d%_]*$") then + return k + else + return '["'..k..'"]' end +end - local function simple_table(t) - if #t > 0 then - local n = 0 - for _,v in pairs(t) do - n = n + 1 - end - if n == #t then - local tt = { } - for i=1,#t do - local v = t[i] - local tv = type(v) - if tv == "number" or tv == "boolean" then - tt[#tt+1] = tostring(v) - elseif tv == "string" then - tt[#tt+1] = ("%q"):format(v) +local function simple_table(t) + if #t > 0 then + local n = 0 + for _,v in pairs(t) do + n = n + 1 + end + if n == #t then + local tt = { } + for i=1,#t do + local v = t[i] + local tv = type(v) + if tv == "number" then + if hexify then + tt[#tt+1] = ("0x%04X"):format(v) else - tt = nil - break + tt[#tt+1] = tostring(v) end + elseif tv == "boolean" then + tt[#tt+1] = tostring(v) + elseif tv == "string" then + tt[#tt+1] = ("%q"):format(v) + else + tt = nil + break end - return tt end + return tt end - return nil end + return nil +end - local function serialize(root,name,handle,depth,level,reduce,noquotes,indexed) - handle = handle or print - reduce = reduce or false - if depth then - depth = depth .. " " - if indexed then - handle(("%s{"):format(depth)) - else - handle(("%s%s={"):format(depth,key(name))) - end - else - depth = "" - local tname = type(name) - if tname == "string" then - if name == "return" then - handle("return {") - else - handle(name .. "={") - end - elseif tname == "number" then - handle("[" .. name .. "]={") - elseif tname == "boolean" then - if name then - handle("return {") - else - handle("{") - end - else - handle("t={") - end - end - if root and next(root) then - local compact = table.serialize_compact - local inline = compact and table.serialize_inline - local first, last = nil, 0 -- #root cannot be trusted here - if compact then - for k,v in ipairs(root) do -- NOT: for k=1,#root do (why) - if not first then first = k end - last = last + 1 - end - end - for _,k in pairs(table.sortedkeys(root)) do - local v = root[k] - local t = type(v) - if compact and first and type(k) == "number" and k >= first and k <= last then - if t == "number" then - handle(("%s %s,"):format(depth,v)) - elseif t == "string" then - if reduce and (v:find("^[%-%+]?[%d]-%.?[%d+]$") == 1) then - handle(("%s %s,"):format(depth,v)) - else - handle(("%s %q,"):format(depth,v)) - end - elseif t == "table" then - if not next(v) then - handle(("%s {},"):format(depth)) - elseif inline then - local st = simple_table(v) - if st then - handle(("%s { %s },"):format(depth,table.concat(st,", "))) - else - serialize(v,k,handle,depth,level+1,reduce,noquotes,true) - end - else - serialize(v,k,handle,depth,level+1,reduce,noquotes,true) - end - elseif t == "boolean" then - handle(("%s %s,"):format(depth,tostring(v))) - elseif t == "function" then - if table.serialize_functions then - handle(('%s loadstring(%q),'):format(depth,string.dump(v))) - else - handle(('%s "function",'):format(depth)) - end +local function do_serialize(root,name,depth,level,indexed) + if level > 0 then + depth = depth .. " " + if indexed then + handle(("%s{"):format(depth)) + elseif name then + handle(("%s%s={"):format(depth,key(name))) + else + handle(("%s{"):format(depth)) + end + end + if root and next(root) then + local first, last = nil, 0 -- #root cannot be trusted here + if compact then + for k,v in ipairs(root) do -- NOT: for k=1,#root do (we need to quit at nil) + if not first then first = k end + last = last + 1 + end + end + --~ for _,k in pairs(sortedkeys(root)) do -- 1% faster: + local sk = sortedkeys(root) + for i=1,#sk do + local k = sk[i] + local v = root[k] + local t = type(v) + if compact and first and type(k) == "number" and k >= first and k <= last then + if t == "number" then + if hexify then + handle(("%s 0x%04X,"):format(depth,v)) else - handle(("%s %q,"):format(depth,tostring(v))) - end - elseif k == "__p__" then -- parent - if false then - handle(("%s __p__=nil,"):format(depth)) + handle(("%s %s,"):format(depth,v)) end - elseif t == "number" then - handle(("%s %s=%s,"):format(depth,key(k),v)) elseif t == "string" then if reduce and (v:find("^[%-%+]?[%d]-%.?[%d+]$") == 1) then - handle(("%s %s=%s,"):format(depth,key(k),v)) + handle(("%s %s,"):format(depth,v)) else - handle(("%s %s=%q,"):format(depth,key(k),v)) + handle(("%s %q,"):format(depth,v)) end elseif t == "table" then if not next(v) then - handle(("%s %s={},"):format(depth,key(k))) + handle(("%s {},"):format(depth)) elseif inline then local st = simple_table(v) if st then - handle(("%s %s={ %s },"):format(depth,key(k),table.concat(st,", "))) + handle(("%s { %s },"):format(depth,concat(st,", "))) else - serialize(v,k,handle,depth,level+1,reduce,noquotes) + do_serialize(v,k,depth,level+1,true) end else - serialize(v,k,handle,depth,level+1,reduce,noquotes) + do_serialize(v,k,depth,level+1,true) end elseif t == "boolean" then - handle(("%s %s=%s,"):format(depth,key(k),tostring(v))) + handle(("%s %s,"):format(depth,tostring(v))) elseif t == "function" then - if table.serialize_functions then - handle(('%s %s=loadstring(%q),'):format(depth,key(k),string.dump(v))) + if functions then + handle(('%s loadstring(%q),'):format(depth,v:dump())) else - handle(('%s %s="function",'):format(depth,key(k))) + handle(('%s "function",'):format(depth)) end else - handle(("%s %s=%q,"):format(depth,key(k),tostring(v))) - -- handle(('%s %s=loadstring(%q),'):format(depth,key(k),string.dump(function() return v end))) + handle(("%s %q,"):format(depth,tostring(v))) + end + elseif k == "__p__" then -- parent + if false then + handle(("%s __p__=nil,"):format(depth)) + end + elseif t == "number" then + if hexify then + handle(("%s %s=0x%04X,"):format(depth,key(k),v)) + else + handle(("%s %s=%s,"):format(depth,key(k),v)) + end + elseif t == "string" then + if reduce and (v:find("^[%-%+]?[%d]-%.?[%d+]$") == 1) then + handle(("%s %s=%s,"):format(depth,key(k),v)) + else + handle(("%s %s=%q,"):format(depth,key(k),v)) + end + elseif t == "table" then + if not next(v) then + handle(("%s %s={},"):format(depth,key(k))) + elseif inline then + local st = simple_table(v) + if st then + handle(("%s %s={ %s },"):format(depth,key(k),concat(st,", "))) + else + do_serialize(v,k,depth,level+1) + end + else + do_serialize(v,k,depth,level+1) + end + elseif t == "boolean" then + handle(("%s %s=%s,"):format(depth,key(k),tostring(v))) + elseif t == "function" then + if functions then + handle(('%s %s=loadstring(%q),'):format(depth,key(k),v:dump())) + else + handle(('%s %s="function",'):format(depth,key(k))) end - end - if level > 0 then - handle(("%s},"):format(depth)) else - handle(("%s}"):format(depth)) + handle(("%s %s=%q,"):format(depth,key(k),tostring(v))) + -- handle(('%s %s=loadstring(%q),'):format(depth,key(k),string.dump(function() return v end))) end - else - handle(("%s}"):format(depth)) end end + if level > 0 then + handle(("%s},"):format(depth)) + end +end - --~ name: - --~ - --~ true : return { } - --~ false : { } - --~ nil : t = { } - --~ string : string = { } - --~ 'return' : return { } - --~ number : [number] = { } - - function table.serialize(root,name,reduce,noquotes) - local t = { } - local function flush(s) - t[#t+1] = s +local function serialize(root,name,_handle,_reduce,_noquotes,_hexify) + noquotes = _noquotes + hexify = _hexify + handle = _handle or print + reduce = _reduce or false + compact = table.serialize_compact + inline = compact and table.serialize_inline + functions = table.serialize_functions + local tname = type(name) + if tname == "string" then + if name == "return" then + handle("return {") + else + handle(name .. "={") + end + elseif tname == "number" then + if hexify then + handle(("[0x%04X]={"):format(name)) + else + handle("[" .. name .. "]={") end - serialize(root, name, flush, nil, 0, reduce, noquotes) - return table.concat(t,"\n") + elseif tname == "boolean" then + if name then + handle("return {") + else + handle("{") + end + else + handle("t={") end + if root and next(root) then + do_serialize(root,name,"",0,indexed) + end + handle("}") +end + +--~ name: +--~ +--~ true : return { } +--~ false : { } +--~ nil : t = { } +--~ string : string = { } +--~ 'return' : return { } +--~ number : [number] = { } - function table.tohandle(handle,root,name,reduce,noquotes) - serialize(root, name, handle, nil, 0, reduce, noquotes) +function table.serialize(root,name,reduce,noquotes,hexify) + local t = { } + local function flush(s) + t[#t+1] = s end + serialize(root,name,flush,reduce,noquotes,hexify) + return concat(t,"\n") +end - -- sometimes tables are real use (zapfino extra pro is some 85M) in which - -- case a stepwise serialization is nice; actually, we could consider: - -- - -- for line in table.serializer(root,name,reduce,noquotes) do - -- ...(line) - -- end - -- - -- so this is on the todo list +function table.tohandle(handle,root,name,reduce,noquotes,hexify) + serialize(root,name,handle,reduce,noquotes,hexify) +end - table.tofile_maxtab = 2*1024 +-- sometimes tables are real use (zapfino extra pro is some 85M) in which +-- case a stepwise serialization is nice; actually, we could consider: +-- +-- for line in table.serializer(root,name,reduce,noquotes) do +-- ...(line) +-- end +-- +-- so this is on the todo list - function table.tofile(filename,root,name,reduce,noquotes) - local f = io.open(filename,'w') - if f then - local concat = table.concat - local maxtab = table.tofile_maxtab - if maxtab > 1 then - local t = { } - local function flush(s) - t[#t+1] = s - if #t > maxtab then - f:write(concat(t,"\n"),"\n") -- hm, write(sometable) should be nice - t = { } - end - end - serialize(root, name, flush, nil, 0, reduce, noquotes) - f:write(concat(t,"\n"),"\n") - else - local function flush(s) - f:write(s,"\n") +table.tofile_maxtab = 2*1024 + +function table.tofile(filename,root,name,reduce,noquotes,hexify) + local f = io.open(filename,'w') + if f then + local maxtab = table.tofile_maxtab + if maxtab > 1 then + local t = { } + local function flush(s) + t[#t+1] = s + if #t > maxtab then + f:write(concat(t,"\n"),"\n") -- hm, write(sometable) should be nice + t = { } end - serialize(root, name, flush, nil, 0, reduce, noquotes) end - f:close() + serialize(root,name,flush,reduce,noquotes,hexify) + f:write(concat(t,"\n"),"\n") + else + local function flush(s) + f:write(s,"\n") + end + serialize(root,name,flush,reduce,noquotes,hexify) end + f:close() end - end ---~ t = { ---~ b = "123", ---~ a = "x", ---~ c = 1.23, ---~ d = "1.23", ---~ e = true, ---~ f = { ---~ d = "1.23", ---~ a = "x", ---~ b = "123", ---~ c = 1.23, ---~ e = true, ---~ f = { ---~ e = true, ---~ f = { ---~ e = true ---~ }, ---~ }, ---~ }, ---~ g = function() end ---~ } - ---~ print(table.serialize(t), "\n") ---~ print(table.serialize(t,"name"), "\n") ---~ print(table.serialize(t,false), "\n") ---~ print(table.serialize(t,true), "\n") ---~ print(table.serialize(t,"name",true), "\n") ---~ print(table.serialize(t,"name",true,true), "\n") - -do - - local function flatten(t,f,complete) - for i=1,#t do - local v = t[i] - if type(v) == "table" then - if complete or type(v[1]) == "table" then - flatten(v,f,complete) - else - f[#f+1] = v - end +local function flatten(t,f,complete) + for i=1,#t do + local v = t[i] + if type(v) == "table" then + if complete or type(v[1]) == "table" then + flatten(v,f,complete) else f[#f+1] = v end + else + f[#f+1] = v end end +end - function table.flatten(t) - local f = { } - flatten(t,f,true) - return f - end +function table.flatten(t) + local f = { } + flatten(t,f,true) + return f +end - function table.unnest(t) -- bad name - local f = { } - flatten(t,f,false) - return f - end +function table.unnest(t) -- bad name + local f = { } + flatten(t,f,false) + return f +end - table.flatten_one_level = table.unnest +table.flatten_one_level = table.unnest +-- the next three may disappear + +function table.remove_value(t,value) -- todo: n + if value then + for i=1,#t do + if t[i] == value then + remove(t,i) + -- remove all, so no: return + end + end + end end function table.insert_before_value(t,value,str) - for i=1,#t do - if t[i] == value then - table.insert(t,i,str) - return + if str then + if value then + for i=1,#t do + if t[i] == value then + insert(t,i,str) + return + end + end end + insert(t,1,str) + elseif value then + insert(t,1,value) end - table.insert(t,1,str) end function table.insert_after_value(t,value,str) - for i=1,#t do - if t[i] == value then - table.insert(t,i+1,str) - return + if str then + if value then + for i=1,#t do + if t[i] == value then + insert(t,i+1,str) + return + end + end end + t[#t+1] = str + elseif value then + t[#t+1] = value end - t[#t+1] = str end function table.are_equal(a,b,n,m) @@ -1026,27 +1059,11 @@ function table.compact(t) end end -function table.tohash(t) - local h = { } - for _, v in pairs(t) do -- no ipairs here - h[v] = true - end - return h -end - -function table.fromhash(t) - local h = { } - for k, v in pairs(t) do -- no ipairs here - if v then h[#h+1] = k end - end - return h -end - function table.contains(t, v) if t then for i=1, #t do if t[i] == v then - return true + return i end end end @@ -1083,11 +1100,10 @@ function table.clone(t,p) -- t is option return t end - function table.hexed(t,seperator) local tt = { } - for i=1,#t do tt[i] = string.format("0x%04X",t[i]) end - return table.concat(tt,seperator or " ") + for i=1,#t do tt[i] = ("0x%04X"):format(t[i]) end + return concat(tt,seperator or " ") end function table.reverse_hash(h) @@ -1127,6 +1143,7 @@ function io.loaddata(filename) local f = io.open(filename,'rb') if f then local data = f:read('*all') + -- garbagecollector.check(data) f:close() return data else @@ -1135,7 +1152,7 @@ function io.loaddata(filename) end function io.savedata(filename,data,joiner) - local f = io.open(filename, "wb") + local f = io.open(filename,"wb") if f then if type(data) == "table" then f:write(table.join(data,joiner or "")) @@ -1145,6 +1162,9 @@ function io.savedata(filename,data,joine f:write(data) end f:close() + return true + else + return false end end @@ -1499,6 +1519,9 @@ end -- copyright: PRAGMA ADE / ConTeXt Development Team -- license : see context related readme files + +--~ print(table.serialize(os.uname())) + if not versions then versions = { } end versions['l-os'] = 1.001 function os.resultof(command) @@ -1574,10 +1597,14 @@ if not versions then versions = { } end if not file then file = { } end +local concat = table.concat + function file.removesuffix(filename) - return filename:gsub("%.[%a%d]+$", "") + return (filename:gsub("%.[%a%d]+$","")) end +file.stripsuffix = file.removesuffix + function file.addsuffix(filename, suffix) if not filename:find("%.[%a%d]+$") then return filename .. "." .. suffix @@ -1587,11 +1614,7 @@ function file.addsuffix(filename, suffix end function file.replacesuffix(filename, suffix) - if not filename:find("%.[%a%d]+$") then - return filename .. "." .. suffix - else - return (filename:gsub("%.[%a%d]+$","."..suffix)) - end + return (filename:gsub("%.[%a%d]+$","")) .. "." .. suffix end function file.dirname(name) @@ -1612,18 +1635,6 @@ end file.suffix = file.extname -function file.stripsuffix(name) - return (name:gsub("%.[%a%d]+$","")) -end - ---~ function file.join(...) ---~ local t = { ... } ---~ for i=1,#t do ---~ t[i] = (t[i]:gsub("\\","/")):gsub("/+$","") ---~ end ---~ return table.concat(t,"/") ---~ end - --~ print(file.join("x/","/y")) --~ print(file.join("http://","/y")) --~ print(file.join("http://a","/y")) @@ -1631,7 +1642,7 @@ end --~ print(file.join("//nas-1","/y")) function file.join(...) - local pth = table.concat({...},"/") + local pth = concat({...},"/") pth = pth:gsub("\\","/") local a, b = pth:match("^(.*://)(.*)$") if a and b then @@ -1664,6 +1675,16 @@ function file.is_readable(name) end end +function file.iswritable(name) + local a = lfs.attributes(name) + return a and a.permissions:sub(2,2) == "w" +end + +function file.isreadable(name) + local a = lfs.attributes(name) + return a and a.permissions:sub(1,1) == "r" +end + --~ function file.split_path(str) --~ if str:find(';') then --~ return str:splitchr(";") @@ -1688,37 +1709,29 @@ function file.split_path(str) end function file.join_path(tab) - return table.concat(tab,io.pathseparator) -- can have trailing // + return concat(tab,io.pathseparator) -- can have trailing // end ---~ print('test' .. " == " .. file.collapse_path("test")) ---~ print("test/test" .. " == " .. file.collapse_path("test/test")) ---~ print("test/test/test" .. " == " .. file.collapse_path("test/test/test")) ---~ print("test/test" .. " == " .. file.collapse_path("test/../test/test")) ---~ print("test" .. " == " .. file.collapse_path("test/../test")) ---~ print("../test" .. " == " .. file.collapse_path("../test")) ---~ print("../test/" .. " == " .. file.collapse_path("../test/")) ---~ print("a/a" .. " == " .. file.collapse_path("a/b/c/../../a")) - ---~ function file.collapse_path(str) ---~ local ok, n = false, 0 ---~ while not ok do ---~ ok = true ---~ str, n = str:gsub("[^%./]+/%.%./", function(s) ---~ ok = false ---~ return "" ---~ end) ---~ end ---~ return (str:gsub("/%./","/")) ---~ end - function file.collapse_path(str) - local n = 1 - while n > 0 do - str, n = str:gsub("([^/%.]+/%.%./)","") - end - return (str:gsub("/%./","/")) -end + str = str:gsub("/%./","/") + local n, m = 1, 1 + while n > 0 or m > 0 do + str, n = str:gsub("[^/%.]+/%.%.$","") + str, m = str:gsub("[^/%.]+/%.%./","") + end + str = str:gsub("([^/])/$","%1") + str = str:gsub("^%./","") + str = str:gsub("/%.$","") + if str == "" then str = "." end + return str +end + +--~ print(file.collapse_path("a/./b/..")) +--~ print(file.collapse_path("a/aa/../b/bb")) +--~ print(file.collapse_path("a/../..")) +--~ print(file.collapse_path("a/.././././b/..")) +--~ print(file.collapse_path("a/./././b/..")) +--~ print(file.collapse_path("a/b/c/../..")) function file.robustname(str) return (str:gsub("[^%a%d%/%-%.\\]+","-")) @@ -1731,6 +1744,98 @@ function file.copy(oldname,newname) file.savedata(newname,io.loaddata(oldname)) end +-- lpeg variants, slightly faster, not always + +--~ local period = lpeg.P(".") +--~ local slashes = lpeg.S("\\/") +--~ local noperiod = 1-period +--~ local noslashes = 1-slashes +--~ local name = noperiod^1 + +--~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.C(noperiod^1) * -1 + +--~ function file.extname(name) +--~ return pattern:match(name) or "" +--~ end + +--~ local pattern = lpeg.Cs(((period * noperiod^1 * -1)/"" + 1)^1) + +--~ function file.removesuffix(name) +--~ return pattern:match(name) +--~ end + +--~ file.stripsuffix = file.removesuffix + +--~ local pattern = (noslashes^0 * slashes)^1 * lpeg.C(noslashes^1) * -1 + +--~ function file.basename(name) +--~ return pattern:match(name) or name +--~ end + +--~ local pattern = (noslashes^0 * slashes)^1 * lpeg.Cp() * noslashes^1 * -1 + +--~ function file.dirname(name) +--~ local p = pattern:match(name) +--~ if p then +--~ return name:sub(1,p-2) +--~ else +--~ return "" +--~ end +--~ end + +--~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1 + +--~ function file.addsuffix(name, suffix) +--~ local p = pattern:match(name) +--~ if p then +--~ return name +--~ else +--~ return name .. "." .. suffix +--~ end +--~ end + +--~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1 + +--~ function file.replacesuffix(name,suffix) +--~ local p = pattern:match(name) +--~ if p then +--~ return name:sub(1,p-2) .. "." .. suffix +--~ else +--~ return name .. "." .. suffix +--~ end +--~ end + +--~ local pattern = (noslashes^0 * slashes)^0 * lpeg.Cp() * ((noperiod^1 * period)^1 * lpeg.Cp() + lpeg.P(true)) * noperiod^1 * -1 + +--~ function file.nameonly(name) +--~ local a, b = pattern:match(name) +--~ if b then +--~ return name:sub(a,b-2) +--~ elseif a then +--~ return name:sub(a) +--~ else +--~ return name +--~ end +--~ end + +--~ local test = file.extname +--~ local test = file.stripsuffix +--~ local test = file.basename +--~ local test = file.dirname +--~ local test = file.addsuffix +--~ local test = file.replacesuffix +--~ local test = file.nameonly + +--~ print(1,test("./a/b/c/abd.def.xxx","!!!")) +--~ print(2,test("./../b/c/abd.def.xxx","!!!")) +--~ print(3,test("a/b/c/abd.def.xxx","!!!")) +--~ print(4,test("a/b/c/def.xxx","!!!")) +--~ print(5,test("a/b/c/def","!!!")) +--~ print(6,test("def","!!!")) +--~ print(7,test("def.xxx","!!!")) + +--~ local tim = os.clock() for i=1,250000 do local ext = test("abd.def.xxx","!!!") end print(os.clock()-tim) + -- filename : l-dir.lua -- comment : split off from luat-lib @@ -1746,51 +1851,6 @@ dir = { } if lfs then do ---~ local attributes = lfs.attributes ---~ local walkdir = lfs.dir ---~ ---~ local function glob_pattern(path,patt,recurse,action) ---~ local ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe ---~ if ok and type(scanner) == "function" then ---~ if not path:find("/$") then path = path .. '/' end ---~ for name in scanner do ---~ local full = path .. name ---~ local mode = attributes(full,'mode') ---~ if mode == 'file' then ---~ if name:find(patt) then ---~ action(full) ---~ end ---~ elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then ---~ glob_pattern(full,patt,recurse,action) ---~ end ---~ end ---~ end ---~ end ---~ ---~ dir.glob_pattern = glob_pattern ---~ ---~ local function glob(pattern, action) ---~ local t = { } ---~ local action = action or function(name) t[#t+1] = name end ---~ local path, patt = pattern:match("^(.*)/*%*%*/*(.-)$") ---~ local recurse = path and patt ---~ if not recurse then ---~ path, patt = pattern:match("^(.*)/(.-)$") ---~ if not (path and patt) then ---~ path, patt = '.', pattern ---~ end ---~ end ---~ patt = patt:gsub("([%.%-%+])", "%%%1") ---~ patt = patt:gsub("%*", ".*") ---~ patt = patt:gsub("%?", ".") ---~ patt = "^" .. patt .. "$" ---~ -- print('path: ' .. path .. ' | pattern: ' .. patt .. ' | recurse: ' .. tostring(recurse)) ---~ glob_pattern(path,patt,recurse,action) ---~ return t ---~ end ---~ ---~ dir.glob = glob - local attributes = lfs.attributes local walkdir = lfs.dir @@ -1819,30 +1879,6 @@ if lfs then do dir.glob_pattern = glob_pattern - --~ local function glob(pattern, action) - --~ local t = { } - --~ local path, rest, patt, recurse - --~ local action = action or function(name) t[#t+1] = name end - --~ local pattern = pattern:gsub("^%*%*","./**") - --~ local pattern = pattern:gsub("/%*/","/**/") - --~ path, rest = pattern:match("^(/)(.-)$") - --~ if path then - --~ path = path - --~ else - --~ path, rest = pattern:match("^([^/]*)/(.-)$") - --~ end - --~ if rest then - --~ patt = rest:gsub("([%.%-%+])", "%%%1") - --~ end - --~ patt = patt:gsub("%*", "[^/]*") - --~ patt = patt:gsub("%?", "[^/]") - --~ patt = patt:gsub("%[%^/%]%*%[%^/%]%*", ".*") - --~ if path == "" then path = "." end - --~ recurse = patt:find("%.%*/") ~= nil - --~ glob_pattern(path,patt,recurse,action) - --~ return t - --~ end - local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V local pattern = Ct { @@ -1868,13 +1904,17 @@ if lfs then do glob(s,t) end return t + elseif lfs.isfile(str) then + local t = t or { } + t[#t+1] = str + return t else local split = pattern:match(str) if split then local t = t or { } local action = action or function(name) t[#t+1] = name end local root, path, base = split[1], split[2], split[3] - local recurse = base:find("**") + local recurse = base:find("%*%*") local start = root .. path local result = filter:match(start .. base) glob_pattern(start,result,recurse,action) @@ -1902,16 +1942,21 @@ if lfs then do for name in walkdir(path) do if name:find("^%.") then --- skip - elseif attributes(name,'mode') == "directory" then - if recurse then - globfiles(path .. "/" .. name,recurse,func,files) - end - elseif func then - if func(name) then - files[#files+1] = path .. "/" .. name - end else - files[#files+1] = path .. "/" .. name + local mode = attributes(name,'mode') + if mode == "directory" then + if recurse then + globfiles(path .. "/" .. name,recurse,func,files) + end + elseif mode == "file" then + if func then + if func(name) then + files[#files+1] = path .. "/" .. name + end + else + files[#files+1] = path .. "/" .. name + end + end end end return files @@ -2119,7 +2164,7 @@ function toboolean(str,tolerant) if tolerant then local tstr = type(str) if tstr == "string" then - return str == "true" or str == "yes" or str == "on" or str == "1" + return str == "true" or str == "yes" or str == "on" or str == "1" or str == "t" elseif tstr == "number" then return tonumber(str) ~= 0 elseif tstr == "nil" then @@ -2138,9 +2183,9 @@ end function string.is_boolean(str) if type(str) == "string" then - if str == "true" or str == "yes" or str == "on" then + if str == "true" or str == "yes" or str == "on" or str == "t" then return true - elseif str == "false" or str == "no" or str == "off" then + elseif str == "false" or str == "no" or str == "off" or str == "f" then return false end end @@ -2156,6 +2201,35 @@ function boolean.falsetrue() end +-- filename : l-math.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-math'] = 1.001 + +local floor = math.floor + +if not math.round then + function math.round(x) + return floor(x + 0.5) + end +end + +if not math.div then + function math.div(n,m) + return floor(n/m) + end +end + +if not math.mod then + function math.mod(n,m) + return n % m + end +end + + if not modules then modules = { } end modules ['l-xml'] = { version = 1.001, comment = "this module is the basis for the lxml-* ones", @@ -2199,11 +2273,11 @@ xml.trace_lpath = false xml.trace_print = false xml.trace_remap = false -local format, concat = string.format, table.concat +local format, concat, remove, insert, type, next = string.format, table.concat, table.remove, table.insert, type, next --~ local pairs, next, type = pairs, next, type --- todo: some things per xml file, liek namespace remapping +-- todo: some things per xml file, like namespace remapping --[[ldx-- <p>First a hack to enable namespace resolving. A namespace is characterized by @@ -2312,7 +2386,7 @@ do -- not just one big nested table capture (lpeg overflow) - local remove, nsremap, resolvens = table.remove, xml.xmlns, xml.resolvens + local nsremap, resolvens = xml.xmlns, xml.resolvens local stack, top, dt, at, xmlns, errorstr, entities = {}, {}, {}, {}, {}, nil, {} @@ -2364,7 +2438,7 @@ do end dt = top.dt dt[#dt+1] = toclose - if at.xmlns then + if toclose.at.xmlns then remove(xmlns) end end @@ -2378,10 +2452,10 @@ do local t = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = top } dt[#dt+1] = t setmetatable(t, mt) - at = { } if at.xmlns then remove(xmlns) end + at = { } end local function add_text(text) if cleanup and #text > 0 then @@ -2638,22 +2712,19 @@ do elseif not nocommands then local ec = e.command if ec ~= nil then -- we can have all kind of types - -if e.special then -- todo test for true/false - local etg, edt = e.tg, e.dt - local spc = specialconverter and specialconverter[etg] - if spc then ---~ print("SPECIAL",etg,table.serialize(specialconverter), spc) - local result = spc(edt[1]) - if result then - handle(result) - return - else - -- no need to handle any further - end - end -end - + if e.special then + local etg, edt = e.tg, e.dt + local spc = specialconverter and specialconverter[etg] + if spc then + local result = spc(edt[1]) + if result then + handle(result) + return + else + -- no need to handle any further + end + end + end local xc = xml.command if xc then xc(e,ec) @@ -2704,17 +2775,7 @@ end end end if ern and xml.trace_remap and ern ~= ens then ---~ if ats then ---~ ats[#ats+1] = format("xmlns:remapped='%s'",ern) ---~ else ---~ ats = { format("xmlns:remapped='%s'",ern) } ---~ end ---~ if ats then ---~ ats[#ats+1] = format("remappedns='%s'",ens or '-') ---~ else ---~ ats = { format("remappedns='%s'",ens or '-') } ---~ end -ens = ern + ens = ern end if ens ~= "" then if edt and #edt > 0 then @@ -2758,7 +2819,16 @@ ens = ern handle("<" .. etg .. ">") end for i=1,#edt do - serialize(edt[i],handle,textconverter,attributeconverter,specialconverter,nocommands) + local ei = edt[i] + if type(ei) == "string" then + if textconverter then + handle(textconverter(ei)) + else + handle(ei) + end + else + serialize(ei,handle,textconverter,attributeconverter,specialconverter,nocommands) + end end -- handle(format("</%s>",etg)) handle("</" .. etg .. ">") @@ -2781,7 +2851,16 @@ ens = ern end else for i=1,#e do - serialize(e[i],handle,textconverter,attributeconverter,specialconverter,nocommands) + local ei = e[i] + if type(ei) == "string" then + if textconverter then + handle(textconverter(ei)) + else + handle(ei) + end + else + serialize(ei,handle,textconverter,attributeconverter,specialconverter,nocommands) + end end end end @@ -2798,8 +2877,8 @@ ens = ern end end if not found then - table.insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } ) - table.insert(dt, 2, "\n" ) + insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } ) + insert(dt, 2, "\n" ) end end end @@ -2811,14 +2890,14 @@ ens = ern function xml.tostring(root) -- 25% overhead due to collecting if root then - if type(root) == 'string' then - return root - elseif next(root) then -- next is faster than type (and >0 test) - local result = { } - serialize(root,function(s) result[#result+1] = s end) - return concat(result,"") + if type(root) == 'string' then + return root + elseif next(root) then -- next is faster than type (and >0 test) + local result = { } + serialize(root,function(s) result[#result+1] = s end) + return concat(result,"") + end end - end return "" end @@ -2883,6 +2962,18 @@ function xml.content(root) -- bugged return (root and root.dt and xml.tostring(root.dt)) or "" end +function xml.isempty(root, pattern) + if pattern == "" or pattern == "*" then + pattern = nil + end + if pattern then + -- todo + return false + else + return not root or not root.dt or #root.dt == 0 or root.dt == "" + end +end + --[[ldx-- <p>The next helper erases an element but keeps the table as it is, and since empty strings are not serialized (effectively) it does @@ -2925,11 +3016,16 @@ of <l n='xpath'/> and since we're not co will explain more about its usage in other documents.</p> --ldx]]-- +local lpathcalls = 0 -- statisctics +local lpathcached = 0 -- statisctics + do - xml.functions = xml.functions or { } + xml.functions = xml.functions or { } + xml.expressions = xml.expressions or { } - local functions = xml.functions + local functions = xml.functions + local expressions = xml.expressions local actions = { [10] = "stay", @@ -2953,34 +3049,32 @@ do [40] = "processing instruction", } - --~ local function make_expression(str) --could also be an lpeg - --~ str = str:gsub("@([a-zA-Z%-_]+)", "(a['%1'] or '')") - --~ str = str:gsub("position%(%)", "i") - --~ str = str:gsub("text%(%)", "t") - --~ str = str:gsub("!=", "~=") - --~ str = str:gsub("([^=!~<>])=([^=!~<>])", "%1==%2") - --~ str = str:gsub("([a-zA-Z%-_]+)%(", "functions.%1(") - --~ return str, loadstring(format("return function(functions,i,a,t) return %s end", str))() - --~ end - -- a rather dumb lpeg local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc - local lp_position = P("position()") / "id" + -- instead of using functions we just parse a few names which saves a call + -- later on + + local lp_position = P("position()") / "ps" + local lp_index = P("index()") / "id" local lp_text = P("text()") / "tx" - local lp_name = P("name()") / "((rt.ns~='' and rt.ns..':'..rt.tg) or '')" - local lp_tag = P("tag()") / "(rt.tg or '')" - local lp_ns = P("ns()") / "(rt.ns or '')" + local lp_name = P("name()") / "(ns~='' and ns..':'..tg)" -- "((rt.ns~='' and rt.ns..':'..rt.tg) or '')" + local lp_tag = P("tag()") / "tg" -- (rt.tg or '') + local lp_ns = P("ns()") / "ns" -- (rt.ns or '') local lp_noequal = P("!=") / "~=" + P("<=") + P(">=") + P("==") local lp_doequal = P("=") / "==" local lp_attribute = P("@") / "" * Cc("(at['") * R("az","AZ","--","__")^1 * Cc("'] or '')") - local lp_function = C(R("az","AZ","--","__")^1) * P("(") / function(t) - if functions[t] then - return "functions." .. t .. "(" + local lp_lua_function = C(R("az","AZ","--","__")^1 * (P(".") * R("az","AZ","--","__")^1)^1) * P("(") / function(t) -- todo: better . handling + return t .. "(" + end + + local lp_function = C(R("az","AZ","--","__")^1) * P("(") / function(t) -- todo: better . handling + if expressions[t] then + return "expressions." .. t .. "(" else - return "functions.error(" + return "expressions.error(" end end @@ -2988,34 +3082,45 @@ do local rparent = lpeg.P(")") local noparent = 1 - (lparent+rparent) local nested = lpeg.P{lparent * (noparent + lpeg.V(1))^0 * rparent} - local value = lpeg.P(lparent * lpeg.C((noparent + nested)^0) * rparent) + local value = lpeg.P(lparent * lpeg.C((noparent + nested)^0) * rparent) -- lpeg.P{"("*C(((1-S("()"))+V(1))^0)*")"} ---~ local value = P { "(" * C(((1 - S("()")) + V(1))^0) * ")" } + -- if we use a dedicated namespace then we don't need to pass rt and k local lp_special = (C(P("name")+P("text")+P("tag"))) * value / function(t,s) - if functions[t] then + if expressions[t] then if s then - return "functions." .. t .. "(rt,k," .. s ..")" + return "expressions." .. t .. "(r,k," .. s ..")" else - return "functions." .. t .. "(rt,k)" + return "expressions." .. t .. "(r,k)" end else - return "functions.error(" .. t .. ")" + return "expressions.error(" .. t .. ")" end end local converter = lpeg.Cs ( ( lp_position + + lp_index + lp_text + lp_name + -- fast one lp_special + lp_noequal + lp_doequal + lp_attribute + + lp_lua_function + lp_function + 1 )^1 ) + -- expressions,root,rootdt,k,e,edt,ns,tg,idx,hsh[tg] or 1 + + local template = [[ + return function(expressions,r,d,k,e,dt,ns,tg,id,ps) + local at, tx = e.at or { }, dt[1] or "" + return %s + end + ]] + local function make_expression(str) str = converter:match(str) - return str, loadstring(format("return function(functions,id,at,tx,rt,k) return %s end", str))() + return str, loadstring(format(template,str))() end local map = { } @@ -3125,7 +3230,7 @@ do local selector = ( instruction + - many + any + +--~ many + any + -- brrr, not here ! parent + stay + dont_position + position + dont_match_one_of_and_eq + dont_match_one_of_and_ne + @@ -3137,6 +3242,7 @@ do has_attribute + has_value + dont_match_one_of + match_one_of + dont_match + match + + many + any + crap + empty ) @@ -3173,7 +3279,7 @@ do return { map[2] } end if m ~= 11 and m ~= 12 and m ~= 13 and m ~= 14 and m ~= 15 and m ~= 16 then - table.insert(map, 1, { 16 }) + insert(map, 1, { 16 }) end -- print((table.serialize(map)):gsub("[ \n]+"," ")) return map @@ -3184,11 +3290,13 @@ do local cache = { } function xml.lpath(pattern,trace) + lpathcalls = lpathcalls + 1 if type(pattern) == "string" then local result = cache[pattern] - if not result then + if result == nil then -- can be false which is valid -) result = compose(pattern) cache[pattern] = result + lpathcached = lpathcached + 1 end if trace or xml.trace_lpath then xml.lshow(result) @@ -3199,6 +3307,10 @@ do end end + function lpath_cached_patterns() + return cache + end + local fallbackreport = (texio and texio.write) or io.write function xml.lshow(pattern,report) @@ -3266,22 +3378,25 @@ functions.</p> do - local functions = xml.functions + local functions = xml.functions + local expressions = xml.expressions - functions.contains = string.find - functions.find = string.find - functions.upper = string.upper - functions.lower = string.lower - functions.number = tonumber - functions.boolean = toboolean + expressions.contains = string.find + expressions.find = string.find + expressions.upper = string.upper + expressions.lower = string.lower + expressions.number = tonumber + expressions.boolean = toboolean - functions.oneof = function(s,...) -- slow + expressions.oneof = function(s,...) -- slow local t = {...} for i=1,#t do if s == t[i] then return true end end return false end - functions.error = function(str) - xml.error_handler("unknown function in lpath expression",str) + + expressions.error = function(str) + xml.error_handler("unknown function in lpath expression",str or "?") return false end + functions.text = function(root,k,n) -- unchecked, maybe one deeper local t = type(t) if t == "string" then @@ -3291,35 +3406,21 @@ do return (rdt and rdt[k]) or root[k] or "" end end - functions.name = function(root,k,n) - -- way too fuzzy - local found - if not k or not n then - local ns, tg = root.rn or root.ns or "", root.tg - if not tg then - for i=1,#root do - local e = root[i] - if type(e) == "table" then - found = e - break - end - end - elseif ns ~= "" then - return ns .. ":" .. tg - else - return tg - end + + functions.name = function(d,k,n) -- ns + tg + local found = false + n = n or 0 + if not k then + -- not found elseif n == 0 then - local e = root[k] - if type(e) ~= "table" then - found = e - end + local dk = d[k] + found = dk and (type(dk) == "table") and dk elseif n < 0 then for i=k-1,1,-1 do - local e = root[i] - if type(e) == "table" then + local di = d[i] + if type(di) == "table" then if n == -1 then - found = e + found = di break else n = n + 1 @@ -3327,12 +3428,11 @@ do end end else ---~ print(k,n,#root) - for i=k+1,#root,1 do - local e = root[i] - if type(e) == "table" then + for i=k+1,#d,1 do + local di = d[i] + if type(di) == "table" then if n == 1 then - found = e + found = di break else n = n - 1 @@ -3347,11 +3447,51 @@ do else return tg end - else - return "" + else + return "" + end + end + + functions.tag = function(d,k,n) -- only tg + local found = false + n = n or 0 + if not k then + -- not found + elseif n == 0 then + local dk = d[k] + found = dk and (type(dk) == "table") and dk + elseif n < 0 then + for i=k-1,1,-1 do + local di = d[i] + if type(di) == "table" then + if n == -1 then + found = di + break + else + n = n + 1 + end + end + end + else + for i=k+1,#d,1 do + local di = d[i] + if type(di) == "table" then + if n == 1 then + found = di + break + else + n = n - 1 + end + end + end end + return (found and found.tg) or "" end + expressions.text = functions.text + expressions.name = functions.name + expressions.tag = functions.tag + local function traverse(root,pattern,handle,reverse,index,parent,wildcard) -- multiple only for tags, not for namespaces if not root then -- error return false @@ -3424,10 +3564,13 @@ do start, stop, step = stop, start, -1 end local idx = 0 + local hsh = { } -- this will slooow down the lot for k=start,stop,step do -- we used to have functions for all but a case is faster local e = rootdt[k] local ns, tg = e.rn or e.ns, e.tg if tg then + -- we can optimize this for simple searches, but it probably does not pay off + hsh[tg] = (hsh[tg] or 0) + 1 idx = idx + 1 if command == 30 then local ns_a, tg_a = action[3], action[4] @@ -3550,7 +3693,7 @@ do end if not action[2] then matched = not matched end if matched then - matched = action[6](functions,idx,e.at or { },edt[1],rootdt,k) + matched = action[6](expressions,root,rootdt,k,e,edt,ns,tg,idx,hsh[tg] or 1) end end if matched then -- combine tg test and at test @@ -4025,10 +4168,10 @@ do local r, d, k, element = m[1], m[2], m[3], m[4] if not before then k = k + 1 end if element.tg then - table.insert(d,k,element) -- untested + insert(d,k,element) -- untested elseif element.dt then for _,v in ipairs(element.dt) do -- i added - table.insert(d,k,v) + insert(d,k,v) k = k + 1 end end @@ -4116,15 +4259,27 @@ do xml.each_element(xmldata, pattern, include) end - function xml.strip_whitespace(root, pattern) + function xml.strip_whitespace(root, pattern, nolines) -- strips all leading and trailing space ! traverse(root, lpath(pattern), function(r,d,k) local dkdt = d[k].dt if dkdt then -- can be optimized local t = { } for i=1,#dkdt do local str = dkdt[i] - if type(str) == "string" and str:find("^[ \n\r\t]*$") then - -- stripped + if type(str) == "string" then + + if str == "" then + -- stripped + else + if nolines then + str = str:gsub("[ \n\r\t]+"," ") + end + if str == "" then + -- stripped + else + t[#t+1] = str + end + end else t[#t+1] = str end @@ -4326,9 +4481,9 @@ original entity is returned.</p> do if unicode and unicode.utf8 then - xml.entities = xml.entities or { } -- xml.entities.handler == function + xml.entities = xml.entities or { } -- xml.entity_handler == function - function xml.entities.handler(e) + function xml.entity_handler(e) return format("[%s]",e) end @@ -4338,8 +4493,6 @@ do if unicode and unicode.utf8 then return char(tonumber(s,16)) end - local entities = xml.entities -- global entities - function utfize(root) local d = root.dt for k=1,#d do @@ -4361,11 +4514,11 @@ do if unicode and unicode.utf8 then if e:find("#x") then return char(tonumber(e:sub(3),16)) else - local ee = entities[e] + local ee = xml.entities[e] -- we cannot shortcut this one (is reloaded) if ee then return ee else - local h = xml.entities.handler + local h = xml.entity_handler return (h and h(e)) or "&" .. e .. ";" end end @@ -4427,6 +4580,13 @@ do if unicode and unicode.utf8 then end end +function xml.statistics() + return { + lpathcalls = lpathcalls, + lpathcached = lpathcached, + } +end + -- xml.set_text_cleanup(xml.show_text_entities) -- xml.set_text_cleanup(xml.resolve_text_entities) @@ -4505,11 +4665,20 @@ function utils.report(...) print(...) end +utils.merger.strip_comment = true + function utils.merger._self_load_(name) local f, data = io.open(name), "" if f then + utils.report("reading merge from %s",name) data = f:read("*all") f:close() + else + utils.report("unknown file to merge %s",name) + end + if data and utils.merger.strip_comment then + -- saves some 20K + data = data:gsub("%-%-~[^\n\r]*[\r\n]", "") end return data or "" end @@ -4518,6 +4687,7 @@ function utils.merger._self_save_(name, if data ~= "" then local f = io.open(name,'w') if f then + utils.report("saving merge from %s",name) f:write(data) f:close() end @@ -4543,13 +4713,13 @@ function utils.merger._self_libs_(libs,l local name = string.gsub(pth .. "/" .. lib,"\\","/") f = io.open(name) if f then - -- utils.report("merging library",name) + utils.report("merging library %s",name) result[#result+1] = f:read("*all") f:close() list = { pth } -- speed up the search break else - -- utils.report("no library",name) + utils.report("no library %s",name) end end end @@ -4588,108 +4758,62 @@ function utils.merger.selfclean(name) ) end -utils.lua.compile_strip = true - -function utils.lua.compile(luafile, lucfile) +function utils.lua.compile(luafile, lucfile, cleanup, strip) -- defaults: cleanup=false strip=true -- utils.report("compiling",luafile,"into",lucfile) os.remove(lucfile) local command = "-o " .. string.quote(lucfile) .. " " .. string.quote(luafile) - if utils.lua.compile_strip then + if strip ~= false then command = "-s " .. command end - if os.spawn("texluac " .. command) == 0 then - return true - elseif os.spawn("luac " .. command) == 0 then - return true - else - return false + local done = (os.spawn("texluac " .. command) == 0) or (os.spawn("luac " .. command) == 0) + if done and cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then + -- utils.report("removing",luafile) + os.remove(luafile) end + return done end --- filename : luat-lib.lua --- comment : companion to luat-lib.tex --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files - -if not versions then versions = { } end versions['luat-lib'] = 1.001 - --- mostcode moved to the l-*.lua and other luat-*.lua files +if not modules then modules = { } end modules ['luat-lib'] = { + version = 1.001, + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files", + comment = "companion to luat-lib.tex", +} --- os / io +-- most code already moved to the l-*.lua and other luat-*.lua files os.setlocale(nil,nil) -- useless feature and even dangerous in luatex --- os.platform - --- mswin|bccwin|mingw|cygwin windows --- darwin|rhapsody|nextstep macosx --- netbsd|unix unix --- linux linux - -if not io.fileseparator then - if string.find(os.getenv("PATH"),";") then - io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" - else - io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" - end -end - -os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix" - --- arg normalization --- --- for k,v in pairs(arg) do print(k,v) end - --- environment - -if not environment then environment = { } end - -environment.ownbin = environment.ownbin or arg[-2] or arg[-1] or arg[0] or "luatex" - -local ownpath = nil -- we could use a metatable here - -function environment.ownpath() - if not ownpath then - for p in string.gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do - local b = file.join(p,environment.ownbin) - if lfs.isfile(b..".exe") or lfs.isfile(b) then - ownpath = p - break - end - end - if not ownpath then ownpath = '.' end - end - return ownpath +function os.setlocale() + -- no way you can mess with it end if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil end -environment.arguments = { } -environment.files = { } -environment.sorted_argument_keys = nil - -environment.platform = os.platform +environment = environment or { } +environment.arguments = { } +environment.files = { } +environment.sortedflags = nil function environment.initialize_arguments(arg) - environment.arguments = { } - environment.files = { } - environment.sorted_argument_keys = nil + local arguments, files = { }, { } + environment.arguments, environment.files, environment.sortedflags = arguments, files, nil for index, argument in pairs(arg) do if index > 0 then local flag, value = argument:match("^%-+(.+)=(.-)$") if flag then - environment.arguments[flag] = string.unquote(value or "") + arguments[flag] = string.unquote(value or "") else flag = argument:match("^%-+(.+)") if flag then - environment.arguments[flag] = true + arguments[flag] = true else - environment.files[#environment.files+1] = argument + files[#files+1] = argument end end end @@ -4710,19 +4834,21 @@ function environment.setargument(name,va environment.arguments[name] = value end -function environment.argument(name) - if environment.arguments[name] then - return environment.arguments[name] - else - if not environment.sorted_argument_keys then - environment.sorted_argument_keys = { } - for _,v in pairs(table.sortedkeys(environment.arguments)) do - table.insert(environment.sorted_argument_keys, "^" .. v) +function environment.argument(name) -- todo: default (plus typecheck on default) + local arguments, sortedflags = environment.arguments, environment.sortedflags + if arguments[name] then + return arguments[name] + else + if not sortedflags then + sortedflags = { } + for _,v in pairs(table.sortedkeys(arguments)) do + sortedflags[#sortedflags+1] = "^" .. v end + environment.sortedflags = sortedflags end - for _,v in pairs(environment.sorted_argument_keys) do + for _,v in ipairs(sortedflags) do if name:find(v) then - return environment.arguments[v:sub(2,#v)] + return arguments[v:sub(2,#v)] end end end @@ -4743,48 +4869,106 @@ function environment.split_arguments(sep return before, after end -function environment.reconstruct_commandline(arg) +--~ function environment.reconstruct_commandline(arg) +--~ if not arg then arg = environment.original_arguments end +--~ local result = { } +--~ for _,a in ipairs(arg) do -- ipairs 1 .. #n +--~ local kk, vv = a:match("^(%-+.-)=(.+)$") +--~ if kk and vv then +--~ if vv:find(" ") then +--~ vv = vv:unquote() +--~ vv = vv:gsub('"','\\"') +--~ result[#result+1] = kk .. "=" .. vv:quote() +--~ else +--~ a = a:unquote() +--~ a = a:gsub('"','\\"') +--~ result[#result+1] = a +--~ end +--~ elseif a:find(" ") then +--~ a = a:unquote() +--~ a = a:gsub('"','\\"') +--~ result[#result+1] = a:quote() +--~ else +--~ result[#result+1] = a +--~ end +--~ end +--~ return table.join(result," ") +--~ end + +function environment.reconstruct_commandline(arg,noquote) if not arg then arg = environment.original_arguments end - local result = { } - for _,a in ipairs(arg) do -- ipairs 1 .. #n - local kk, vv = a:match("^(%-+.-)=(.+)$") - if kk and vv then - if vv:find(" ") then - result[#result+1] = kk .. "=" .. string.quote(vv) + if noquote and #arg == 1 then + local a = arg[1] + a = input.resolve(a) + a = a:unquote() + return a + elseif #arg == 1 then + local result = { } + for _,a in ipairs(arg) do -- ipairs 1 .. #n + a = input.resolve(a) + a = a:unquote() + a = a:gsub('"','\\"') -- tricky + if a:find(" ") then + result[#result+1] = a:quote() else result[#result+1] = a end - elseif a:find(" ") then - result[#result+1] = string.quote(a) - else - result[#result+1] = a end + return table.join(result," ") end - return table.join(result," ") end if arg then - environment.initialize_arguments(arg) - environment.original_arguments = arg + + -- new, reconstruct quoted snippets (maybe better just remnove the " then and add them later) + local newarg, instring = { }, false + + for index, argument in ipairs(arg) do + if argument:find("^\"") then + newarg[#newarg+1] = argument:gsub("^\"","") + if not argument:find("\"$") then + instring = true + end + elseif argument:find("\"$") then + newarg[#newarg] = newarg[#newarg] .. " " .. argument:gsub("\"$","") + instring = false + elseif instring then + newarg[#newarg] = newarg[#newarg] .. " " .. argument + else + newarg[#newarg+1] = argument + end + end + for i=1,-5,-1 do + newarg[i] = arg[i] + end + + environment.initialize_arguments(newarg) + environment.original_arguments = newarg + environment.raw_arguments = arg + arg = { } -- prevent duplicate handling -end +end --- filename : luat-inp.lua --- comment : companion to luat-lib.tex --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files --- This lib is multi-purpose and can be loaded again later on so that --- additional functionality becomes available. We will split this --- module in components when we're done with prototyping. +if not modules then modules = { } end modules ['luat-inp'] = { + version = 1.001, + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files", + comment = "companion to luat-lib.tex", +} -- TODO: os.getenv -> os.env[] -- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller) -- TODO: check escaping in find etc, too much, too slow --- This is the first code I wrote for LuaTeX, so it needs some cleanup. +-- This lib is multi-purpose and can be loaded again later on so that +-- additional functionality becomes available. We will split this +-- module in components once we're done with prototyping. This is the +-- first code I wrote for LuaTeX, so it needs some cleanup. Before changing +-- something in this module one can best check with Taco or Hans first; there +-- is some nasty trickery going on that relates to traditional kpse support. -- To be considered: hash key lowercase, first entry in table filename -- (any case), rest paths (so no need for optimization). Or maybe a @@ -4794,12 +4978,6 @@ end -- Beware, loading and saving is overloaded in luat-tmp! -if not versions then versions = { } end versions['luat-inp'] = 1.001 -if not environment then environment = { } end -if not file then file = { } end - -if environment.aleph_mode == nil then environment.aleph_mode = true end -- temp hack - if not input then input = { } end if not input.suffixes then input.suffixes = { } end if not input.formats then input.formats = { } end @@ -4812,7 +4990,7 @@ if not input.hashers then input.hashe if not input.generators then input.generators = { } end -- generate databases if not input.filters then input.filters = { } end -- conversion filters -local format = string.format +local format, concat, sortedkeys = string.format, table.concat, table.sortedkeys input.locators.notfound = { nil } input.hashers.notfound = { nil } @@ -4825,8 +5003,16 @@ input.debug = false input.cnfname = 'texmf.cnf' input.luaname = 'texmfcnf.lua' input.lsrname = 'ls-R' -input.luasuffix = '.tma' -input.lucsuffix = '.tmc' +input.homedir = os.env[os.platform == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~' + +--~ input.luasuffix = 'tma' +--~ input.lucsuffix = 'tmc' + +-- for the moment we have .local but this will disappear +input.cnfdefault = '{$SELFAUTOLOC,$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}' + +-- chances are low that the cnf file is in the bin path +input.cnfdefault = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}' -- we use a cleaned up list / format=any is a wildcard, as is *name @@ -4862,7 +5048,8 @@ input.suffixes['lua'] = { 'lua', 'luc', -- FONTFEATURES = .;$TEXMF/fonts/fea// -- FONTCIDMAPS = .;$TEXMF/fonts/cid// -function input.checkconfigdata(instance) -- not yet ok, no time for debugging now +function input.checkconfigdata() -- not yet ok, no time for debugging now + local instance = input.instance local function fix(varname,default) local proname = varname .. "." .. instance.progname or "crap" local p = instance.environment[proname] @@ -4871,7 +5058,15 @@ function input.checkconfigdata(instance) instance.variables[varname] = default -- or environment? end end - fix("LUAINPUTS" , ".;$TEXINPUTS;$TEXMFSCRIPTS") + local name = os.name + if name == "windows" then + fix("OSFONTDIR", "c:/windows/fonts//") + elseif name == "macosx" then + fix("OSFONTDIR", "$HOME/Library/Fonts//;/Library/Fonts//;/System/Library/Fonts//") + else + -- bad luck + end + fix("LUAINPUTS" , ".;$TEXINPUTS;$TEXMFSCRIPTS") -- no progname, hm fix("FONTFEATURES", ".;$TEXMF/fonts/fea//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") fix("FONTCIDMAPS" , ".;$TEXMF/fonts/cid//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") end @@ -4898,14 +5093,20 @@ input.formats ['sfd'] input.suffixes ['sfd'] = { 'sfd' } input.alternatives['subfont definition files'] = 'sfd' -function input.reset() +-- In practice we will work within one tds tree, but i want to keep +-- the option open to build tools that look at multiple trees, which is +-- why we keep the tree specific data in a table. We used to pass the +-- instance but for practical pusposes we now avoid this and use a +-- instance variable. + +function input.newinstance() local instance = { } instance.rootpath = '' instance.treepath = '' - instance.progname = environment.progname or 'context' - instance.engine = environment.engine or 'luatex' + instance.progname = 'context' + instance.engine = 'luatex' instance.format = '' instance.environment = { } instance.variables = { } @@ -4929,12 +5130,12 @@ function input.reset() instance.cachepath = nil instance.loaderror = false instance.smallcache = false + instance.sortdata = false instance.savelists = true instance.cleanuppaths = true instance.allresults = false instance.pattern = nil -- lists instance.kpseonly = false -- lists - instance.cachefile = 'tmftools' instance.loadtime = 0 instance.starttime = 0 instance.stoptime = 0 @@ -4945,23 +5146,13 @@ function input.reset() instance.fakepaths = { } instance.lsrmode = false - if os.env then - -- store once, freeze and faster - for k,v in pairs(os.env) do - instance.environment[k] = input.bare_variable(v) - end - else - -- we will access os.env frequently - for k,v in pairs({'HOME','TEXMF','TEXMFCNF'}) do - local e = os.getenv(v) - if e then - -- input.report("setting",v,"to",input.bare_variable(e)) - instance.environment[v] = input.bare_variable(e) - end - end + -- store once, freeze and faster (once reset we can best use instance.environment) + + for k,v in pairs(os.env) do + instance.environment[k] = input.bare_variable(v) end - -- cross referencing + -- cross referencing, delayed because we can add suffixes for k, v in pairs(input.suffixes) do for _, vv in pairs(v) do @@ -4975,68 +5166,42 @@ function input.reset() end -function input.reset_hashes(instance) - instance.lists = { } - instance.found = { } -end - -function input.bare_variable(str) -- assumes str is a string - -- return string.gsub(string.gsub(string.gsub(str,"%s+$",""),'^"(.+)"$',"%1"),"^'(.+)'$","%1") - return (str:gsub("\s*([\"\']?)(.+)%1\s*", "%2")) -end +input.instance = input.instance or nil -if texio then - input.log = texio.write_nl -else - input.log = print +function input.reset() + input.instance = input.newinstance() + return input.instance end -function input.simple_logger(kind, name) - if name and name ~= "" then - if input.banner then - input.log(input.banner..kind..": "..name) - else - input.log("<<"..kind..": "..name..">>") - end - else - if input.banner then - input.log(input.banner..kind..": no name") - else - input.log("<<"..kind..": no name>>") - end - end +function input.reset_hashes() + input.instance.lists = { } + input.instance.found = { } end -function input.dummy_logger() +function input.bare_variable(str) -- assumes str is a string + -- return string.gsub(string.gsub(string.gsub(str,"%s+$",""),'^"(.+)"$',"%1"),"^'(.+)'$","%1") + return (str:gsub("\s*([\"\']?)(.+)%1\s*", "%2")) end function input.settrace(n) input.trace = tonumber(n or 0) if input.trace > 0 then - input.logger = input.simple_logger input.verbose = true - else - input.logger = function() end end end -function input.report(...) -- inefficient +input.log = (texio and texio.write_nl) or print + +function input.report(...) if input.verbose then - if input.banner then - input.log(input.banner .. table.concat({...},' ')) - elseif input.logmode() == 'xml' then - input.log("<t>"..table.concat({...},' ').."</t>") - else - input.log("<<"..table.concat({...},' ')..">>") - end + input.log("<<"..format(...)..">>") end end -function input.reportlines(str) - if type(str) == "string" then - str = str:split("\n") +function input.report(...) + if input.trace > 0 then -- extra test + input.log("<<"..format(...)..">>") end - for _,v in pairs(str) do input.report(v) end end input.settrace(tonumber(os.getenv("MTX.INPUT.TRACE") or os.getenv("MTX_INPUT_TRACE") or input.trace or 0)) @@ -5065,7 +5230,7 @@ do instance.stoptime = stoptime instance.loadtime = instance.loadtime + loadtime if report then - input.report('load time', format("%0.3f",loadtime)) + input.report("load time %0.3f",loadtime) end return loadtime end @@ -5081,18 +5246,18 @@ end function input.report_loadtime(instance) if instance then - input.report('total load time', input.elapsedtime(instance)) + input.report('total load time %s', input.elapsedtime(instance)) end end input.loadtime = input.elapsedtime -function input.env(instance,key) - return instance.environment[key] or input.osenv(instance,key) +function input.env(key) + return input.instance.environment[key] or input.osenv(key) end -function input.osenv(instance,key) - local ie = instance.environment +function input.osenv(key) + local ie = input.instance.environment local value = ie[key] if value == nil then -- local e = os.getenv(key) @@ -5110,81 +5275,106 @@ end -- we follow a rather traditional approach: -- -- (1) texmf.cnf given in TEXMFCNF --- (2) texmf.cnf searched in TEXMF/web2c +-- (2) texmf.cnf searched in default variable -- --- for the moment we don't expect a configuration file in a zip +-- also we now follow the stupid route: if not set then just assume *one* +-- cnf file under texmf (i.e. distribution) -function input.identify_cnf(instance) - -- we no longer support treepath and rootpath (was handy for testing); - -- also we now follow the stupid route: if not set then just assume *one* - -- cnf file under texmf (i.e. distribution) - if #instance.cnffiles == 0 then - if input.env(instance,'TEXMFCNF') == "" then - local ownpath = environment.ownpath() or "." - if ownpath then - -- beware, this is tricky on my own system because at that location I do have - -- the raw tree that ends up in the zip; i.e. I cannot test this kind of mess - local function locate(filename,list) - local ownroot = input.normalize_name(file.join(ownpath,"../..")) - if not lfs.isdir(file.join(ownroot,"texmf")) then - ownroot = input.normalize_name(file.join(ownpath,"..")) - if not lfs.isdir(file.join(ownroot,"texmf")) then - input.verbose = true - input.report("error", "unable to identify cnf file") - return +input.ownpath = input.ownpath or nil +input.ownbin = input.ownbin or arg[-2] or arg[-1] or arg[0] or "luatex" +input.autoselfdir = true -- false may be handy for debugging + +function input.getownpath() + if not input.ownpath then + if input.autoselfdir and os.selfdir then + input.ownpath = os.selfdir + else + local binary = input.ownbin + if os.platform == "windows" then + binary = file.replacesuffix(binary,"exe") + end + for p in string.gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do + local b = file.join(p,binary) + if lfs.isfile(b) then + -- we assume that after changing to the path the currentdir function + -- resolves to the real location and use this side effect here; this + -- trick is needed because on the mac installations use symlinks in the + -- path instead of real locations + local olddir = lfs.currentdir() + if lfs.chdir(p) then + local pp = lfs.currentdir() + if input.verbose and p ~= pp then + input.report("following symlink %s to %s",p,pp) end - end - local texmfcnf = file.join(ownroot,"texmf-local/web2c",filename) -- for minimals and myself - if not lfs.isfile(texmfcnf) then - texmfcnf = file.join(ownroot,"texmf/web2c",filename) - if not lfs.isfile(texmfcnf) then - input.verbose = true - input.report("error", "unable to locate",filename) - return + input.ownpath = pp + lfs.chdir(olddir) + else + if input.verbose then + input.report("unable to check path %s",p) end + input.ownpath = p end - table.insert(list,texmfcnf) - local ie = instance.environment - if not ie['SELFAUTOPARENT'] then ie['SELFAUTOPARENT'] = ownroot end - if not ie['TEXMFCNF'] then ie['TEXMFCNF'] = file.dirname(texmfcnf) end - end - locate(input.luaname,instance.luafiles) - locate(input.cnfname,instance.cnffiles) - if #instance.luafiles == 0 and instance.cnffiles == 0 then - input.verbose = true - input.report("error", "unable to locate",filename) - os.exit() + break end - -- here we also assume then TEXMF is set in the distribution, if this trickery is - -- used in the minimals, then users who don't use setuptex are on their own with - -- regards to extra trees - else - input.verbose = true - input.report("error", "unable to identify own path") - os.exit() end - else - local t = input.split_path(input.env(instance,'TEXMFCNF')) - t = input.aux.expanded_path(instance,t) - input.aux.expand_vars(instance,t) - local function locate(filename,list) - for _,v in ipairs(t) do - local texmfcnf = input.normalize_name(file.join(v,filename)) - if lfs.isfile(texmfcnf) then - table.insert(list,texmfcnf) - end + end + if not input.ownpath then input.ownpath = '.' end + end + return input.ownpath +end + +function input.identify_own() + local instance = input.instance + local ownpath = input.getownpath() or lfs.currentdir() + local ie = instance.environment + if ownpath then + if input.env('SELFAUTOLOC') == "" then os.env['SELFAUTOLOC'] = file.collapse_path(ownpath) end + if input.env('SELFAUTODIR') == "" then os.env['SELFAUTODIR'] = file.collapse_path(ownpath .. "/..") end + if input.env('SELFAUTOPARENT') == "" then os.env['SELFAUTOPARENT'] = file.collapse_path(ownpath .. "/../..") end + else + input.verbose = true + input.report("error: unable to locate ownpath") + os.exit() + end + if input.env('TEXMFCNF') == "" then os.env['TEXMFCNF'] = input.cnfdefault end + if input.env('TEXOS') == "" then os.env['TEXOS'] = input.env('SELFAUTODIR') end + if input.env('TEXROOT') == "" then os.env['TEXROOT'] = input.env('SELFAUTOPARENT') end + if input.verbose then + for _,v in ipairs({"SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF"}) do + input.report("variable %s set to %s",v,input.env(v) or "unknown") + end + end + function input.identify_own() end +end + +function input.identify_cnf() + local instance = input.instance + if #instance.cnffiles == 0 then + -- fallback + input.identify_own() + -- the real search + input.expand_variables() + local t = input.split_path(input.env('TEXMFCNF')) + t = input.aux.expanded_path(t) + input.aux.expand_vars(t) -- redundant + local function locate(filename,list) + for _,v in ipairs(t) do + local texmfcnf = input.normalize_name(file.join(v,filename)) + if lfs.isfile(texmfcnf) then + table.insert(list,texmfcnf) end end - locate(input.luaname,instance.luafiles) - locate(input.cnfname,instance.cnffiles) end + locate(input.luaname,instance.luafiles) + locate(input.cnfname,instance.cnffiles) end end -function input.load_cnf(instance) +function input.load_cnf() + local instance = input.instance local function loadoldconfigdata() for _, fname in ipairs(instance.cnffiles) do - input.aux.load_cnf(instance,fname) + input.aux.load_cnf(fname) end end -- instance.cnffiles contain complete names now ! @@ -5199,27 +5389,27 @@ function input.load_cnf(instance) instance.rootpath = file.dirname(instance.rootpath) end instance.rootpath = input.normalize_name(instance.rootpath) - instance.environment['SELFAUTOPARENT'] = instance.rootpath -- just to be sure if instance.lsrmode then loadoldconfigdata() elseif instance.diskcache and not instance.renewcache then - input.loadoldconfig(instance,instance.cnffiles) + input.loadoldconfig(instance.cnffiles) if instance.loaderror then loadoldconfigdata() - input.saveoldconfig(instance) + input.saveoldconfig() end else loadoldconfigdata() if instance.renewcache then - input.saveoldconfig(instance) + input.saveoldconfig() end end - input.aux.collapse_cnf_data(instance) + input.aux.collapse_cnf_data() end - input.checkconfigdata(instance) + input.checkconfigdata() end -function input.load_lua(instance) +function input.load_lua() + local instance = input.instance if #instance.luafiles == 0 then -- yet harmless else @@ -5231,14 +5421,14 @@ function input.load_lua(instance) instance.rootpath = file.dirname(instance.rootpath) end instance.rootpath = input.normalize_name(instance.rootpath) - instance.environment['SELFAUTOPARENT'] = instance.rootpath -- just to be sure - input.loadnewconfig(instance) - input.aux.collapse_cnf_data(instance) + input.loadnewconfig() + input.aux.collapse_cnf_data() end - input.checkconfigdata(instance) + input.checkconfigdata() end -function input.aux.collapse_cnf_data(instance) -- potential optmization: pass start index (setup and configuration are shared) +function input.aux.collapse_cnf_data() -- potential optimization: pass start index (setup and configuration are shared) + local instance = input.instance for _,c in ipairs(instance.order) do for k,v in pairs(c) do if not instance.variables[k] then @@ -5253,21 +5443,22 @@ function input.aux.collapse_cnf_data(ins end end -function input.aux.load_cnf(instance,fname) +function input.aux.load_cnf(fname) + local instance = input.instance fname = input.clean_path(fname) - local lname = fname:gsub("%.%a+$",input.luasuffix) + local lname = file.replacesuffix(fname,'lua') local f = io.open(lname) if f then -- this will go f:close() local dname = file.dirname(fname) if not instance.configuration[dname] then - input.aux.load_configuration(instance,dname,lname) + input.aux.load_configuration(dname,lname) instance.order[#instance.order+1] = instance.configuration[dname] end else f = io.open(fname) if f then - input.report("loading", fname) + input.report("loading %s", fname) local line, data, n, k, v local dname = file.dirname(fname) if not instance.configuration[dname] then @@ -5299,227 +5490,226 @@ function input.aux.load_cnf(instance,fna end f:close() else - input.report("skipping", fname) + input.report("skipping %s", fname) end end end -- database loading -function input.load_hash(instance) - input.locatelists(instance) +function input.load_hash() + local instance = input.instance + input.locatelists() if instance.lsrmode then - input.loadlists(instance) + input.loadlists() elseif instance.diskcache and not instance.renewcache then - input.loadfiles(instance) + input.loadfiles() if instance.loaderror then - input.loadlists(instance) - input.savefiles(instance) + input.loadlists() + input.savefiles() end else - input.loadlists(instance) + input.loadlists() if instance.renewcache then - input.savefiles(instance) + input.savefiles() end end end -function input.aux.append_hash(instance,type,tag,name) - input.logger("= hash append",tag) - table.insert(instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } ) +function input.aux.append_hash(type,tag,name) + if input.trace > 0 then + input.logger("= hash append: %s",tag) + end + table.insert(input.instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } ) end -function input.aux.prepend_hash(instance,type,tag,name) - input.logger("= hash prepend",tag) - table.insert(instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } ) +function input.aux.prepend_hash(type,tag,name) + if input.trace > 0 then + input.logger("= hash prepend: %s",tag) + end + table.insert(input.instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } ) end -function input.aux.extend_texmf_var(instance,specification) -- crap - if instance.environment['TEXMF'] then - input.report("extending environment variable TEXMF with", specification) - instance.environment['TEXMF'] = instance.environment['TEXMF']:gsub("^%{", function() - return "{" .. specification .. "," - end) - elseif instance.variables['TEXMF'] then - input.report("extending configuration variable TEXMF with", specification) - instance.variables['TEXMF'] = instance.variables['TEXMF']:gsub("^%{", function() - return "{" .. specification .. "," - end) +function input.aux.extend_texmf_var(specification) -- crap, we could better prepend the hash + local instance = input.instance +-- local t = input.expanded_path_list('TEXMF') -- full expansion + local t = input.split_path(input.env('TEXMF')) + table.insert(t,1,specification) + local newspec = table.join(t,";") + if instance.environment["TEXMF"] then + instance.environment["TEXMF"] = newspec + elseif instance.variables["TEXMF"] then + instance.variables["TEXMF"] = newspec else - input.report("setting configuration variable TEXMF to", specification) - instance.variables['TEXMF'] = "{" .. specification .. "}" + -- weird end - if instance.variables['TEXMF']:find("%,") and not instance.variables['TEXMF']:find("^%{") then - input.report("adding {} to complex TEXMF variable, best do that yourself") - instance.variables['TEXMF'] = "{" .. instance.variables['TEXMF'] .. "}" - end - input.expand_variables(instance) - input.reset_hashes(instance) + input.expand_variables() + input.reset_hashes() end -- locators -function input.locatelists(instance) - for _, path in pairs(input.simplified_list(input.expansion(instance,'TEXMF'))) do - path = file.collapse_path(path) - input.report("locating list of",path) - input.locatedatabase(instance,input.normalize_name(path)) +function input.locatelists() + local instance = input.instance + for _, path in pairs(input.clean_path_list('TEXMF')) do + input.report("locating list of %s",path) + input.locatedatabase(input.normalize_name(path)) end end -function input.locatedatabase(instance,specification) - return input.methodhandler('locators', instance, specification) +function input.locatedatabase(specification) + return input.methodhandler('locators', specification) end -function input.locators.tex(instance,specification) +function input.locators.tex(specification) if specification and specification ~= '' and lfs.isdir(specification) then - input.logger('! tex locator', specification..' found') - input.aux.append_hash(instance,'file',specification,filename) - else - input.logger('? tex locator', specification..' not found') + if input.trace > 0 then + input.logger('! tex locator found: %s',specification) + end + input.aux.append_hash('file',specification,filename) + elseif input.trace > 0 then + input.logger('? tex locator not found: %s',specification) end end -- hashers -function input.hashdatabase(instance,tag,name) - return input.methodhandler('hashers',instance,tag,name) +function input.hashdatabase(tag,name) + return input.methodhandler('hashers',tag,name) end -function input.loadfiles(instance) +function input.loadfiles() + local instance = input.instance instance.loaderror = false instance.files = { } if not instance.renewcache then for _, hash in ipairs(instance.hashes) do - input.hashdatabase(instance,hash.tag,hash.name) + input.hashdatabase(hash.tag,hash.name) if instance.loaderror then break end end end end -function input.hashers.tex(instance,tag,name) - input.aux.load_files(instance,tag) +function input.hashers.tex(tag,name) + input.aux.load_files(tag) end -- generators: -function input.loadlists(instance) - for _, hash in ipairs(instance.hashes) do - input.generatedatabase(instance,hash.tag) +function input.loadlists() + for _, hash in ipairs(input.instance.hashes) do + input.generatedatabase(hash.tag) end end -function input.generatedatabase(instance,specification) - return input.methodhandler('generators', instance, specification) -end - -do - - local weird = lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) - - function input.generators.tex(instance,specification) - local tag = specification - if not instance.lsrmode and lfs and lfs.dir then - input.report("scanning path",specification) - instance.files[tag] = { } - local files = instance.files[tag] - local n, m, r = 0, 0, 0 - local spec = specification .. '/' - local attributes = lfs.attributes - local directory = lfs.dir - local small = instance.smallcache - local function action(path) - local mode, full - if path then - full = spec .. path .. '/' +function input.generatedatabase(specification) + return input.methodhandler('generators', specification) +end + +local weird = lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) + +function input.generators.tex(specification) + local instance = input.instance + local tag = specification + if not instance.lsrmode and lfs.dir then + input.report("scanning path %s",specification) + instance.files[tag] = { } + local files = instance.files[tag] + local n, m, r = 0, 0, 0 + local spec = specification .. '/' + local attributes = lfs.attributes + local directory = lfs.dir + local small = instance.smallcache + local function action(path) + local mode, full + if path then + full = spec .. path .. '/' + else + full = spec + end + for name in directory(full) do + if name:find("^%.") then + -- skip + -- elseif name:find("[%~%`%!%#%$%%%^%&%*%(%)%=%{%}%[%]%:%;\"\'%|%<%>%,%?\n\r\t]") then -- too much escaped + elseif weird:match(name) then + -- texio.write_nl("skipping " .. name) + -- skip else - full = spec - end - for name in directory(full) do - if name:find("^%.") then - -- skip - -- elseif name:find("[%~%`%!%#%$%%%^%&%*%(%)%=%{%}%[%]%:%;\"\'%|%<%>%,%?\n\r\t]") then -- too much escaped - elseif weird:match(name) then - -- texio.write_nl("skipping " .. name) - -- skip - else - mode = attributes(full..name,'mode') - if mode == "directory" then - m = m + 1 - if path then - action(path..'/'..name) - else - action(name) - end - elseif path and mode == 'file' then - n = n + 1 - local f = files[name] - if f then - if not small then - if type(f) == 'string' then - files[name] = { f, path } - else - f[#f+1] = path - end - end - else - files[name] = path - local lower = name:lower() - if name ~= lower then - files["remap:"..lower] = name - r = r + 1 - end - end + mode = attributes(full..name,'mode') + if mode == 'directory' then + m = m + 1 + if path then + action(path..'/'..name) + else + action(name) end - end - end - end - action() - input.report(format("%s files found on %s directories with %s uppercase remappings",n,m,r)) - else - local fullname = file.join(specification,input.lsrname) - local path = '.' - local f = io.open(fullname) - if f then - instance.files[tag] = { } - local files = instance.files[tag] - local small = instance.smallcache - input.report("loading lsr file",fullname) - -- for line in f:lines() do -- much slower then the next one - for line in (f:read("*a")):gmatch("(.-)\n") do - if line:find("^[%a%d]") then - local fl = files[line] - if fl then + elseif path and mode == 'file' then + n = n + 1 + local f = files[name] + if f then if not small then - if type(fl) == 'string' then - files[line] = { fl, path } -- table + if type(f) == 'string' then + files[name] = { f, path } else - fl[#fl+1] = path + f[#f+1] = path end end else - files[line] = path -- string - local lower = line:lower() - if line ~= lower then - files["remap:"..lower] = line + files[name] = path + local lower = name:lower() + if name ~= lower then + files["remap:"..lower] = name + r = r + 1 + end + end + end + end + end + end + action() + input.report("%s files found on %s directories with %s uppercase remappings",n,m,r) + else + local fullname = file.join(specification,input.lsrname) + local path = '.' + local f = io.open(fullname) + if f then + instance.files[tag] = { } + local files = instance.files[tag] + local small = instance.smallcache + input.report("loading lsr file %s",fullname) + -- for line in f:lines() do -- much slower then the next one + for line in (f:read("*a")):gmatch("(.-)\n") do + if line:find("^[%a%d]") then + local fl = files[line] + if fl then + if not small then + if type(fl) == 'string' then + files[line] = { fl, path } -- table + else + fl[#fl+1] = path end end else - path = line:match("%.%/(.-)%:$") or path -- match could be nil due to empty line + files[line] = path -- string + local lower = line:lower() + if line ~= lower then + files["remap:"..lower] = line + end end + else + path = line:match("%.%/(.-)%:$") or path -- match could be nil due to empty line end - f:close() end + f:close() end end - end -- savers, todo -function input.savefiles(instance) - input.aux.save_data(instance, 'files', function(k,v) - return instance.validfile(k,v) -- path, name +function input.savefiles() + input.aux.save_data('files', function(k,v) + return input.instance.validfile(k,v) -- path, name end) end @@ -5527,8 +5717,8 @@ end -- we join them and split them after the expansion has taken place. This -- is more convenient. -function input.splitconfig(instance) - for i,c in ipairs(instance) do +function input.splitconfig() + for i,c in ipairs(input.instance) do for k,v in pairs(c) do if type(v) == 'string' then local t = file.split_path(v) @@ -5539,8 +5729,9 @@ function input.splitconfig(instance) end end end -function input.joinconfig(instance) - for i,c in ipairs(instance.order) do + +function input.joinconfig() + for i,c in ipairs(input.instance.order) do for k,v in pairs(c) do if type(v) == 'table' then c[k] = file.join_path(v) @@ -5563,8 +5754,9 @@ function input.join_path(str) end end -function input.splitexpansions(instance) - for k,v in pairs(instance.expansions) do +function input.splitexpansions() + local ie = input.instance.expansions + for k,v in pairs(ie) do local t, h = { }, { } for _,vv in pairs(file.split_path(v)) do if vv ~= "" and not h[vv] then @@ -5573,19 +5765,19 @@ function input.splitexpansions(instance) end end if #t > 1 then - instance.expansions[k] = t + ie[k] = t else - instance.expansions[k] = t[1] + ie[k] = t[1] end end end -- end of split/join code -function input.saveoldconfig(instance) - input.splitconfig(instance) - input.aux.save_data(instance, 'configuration', nil) - input.joinconfig(instance) +function input.saveoldconfig() + input.splitconfig() + input.aux.save_data('configuration', nil) + input.joinconfig() end input.configbanner = [[ @@ -5602,8 +5794,6 @@ function input.serialize(files) -- luatools and mtxtools are called frequently. Okay, -- we pay a small price for properly tabbed tables. local t = { } - local concat = table.concat - local sorted = table.sortedkeys local function dump(k,v,m) if type(v) == 'string' then return m .. "['" .. k .. "']='" .. v .. "'," @@ -5614,12 +5804,12 @@ function input.serialize(files) end end t[#t+1] = "return {" - if instance.sortdata then - for _, k in pairs(sorted(files)) do + if input.instance.sortdata then + for _, k in pairs(sortedkeys(files)) do local fk = files[k] if type(fk) == 'table' then t[#t+1] = "\t['" .. k .. "']={" - for _, kk in pairs(sorted(fk)) do + for _, kk in pairs(sortedkeys(fk)) do t[#t+1] = dump(kk,fk[kk],"\t\t") end t[#t+1] = "\t}," @@ -5646,11 +5836,11 @@ end if not texmf then texmf = {} end -- no longer needed, at least not here -function input.aux.save_data(instance, dataname, check, makename) -- untested without cache overload - for cachename, files in pairs(instance[dataname]) do +function input.aux.save_data(dataname, check, makename) -- untested without cache overload + for cachename, files in pairs(input.instance[dataname]) do local name = (makename or file.join)(cachename,dataname) - local luaname, lucname = name .. input.luasuffix, name .. input.lucsuffix - input.report("preparing " .. dataname .. " for", luaname) + local luaname, lucname = name .. ".lua", name .. ".luc" + input.report("preparing %s for %s",dataname,cachename) for k, v in pairs(files) do if not check or check(v,k) then -- path, name if type(v) == "table" and #v == 1 then @@ -5668,38 +5858,38 @@ function input.aux.save_data(instance, d time = os.date("%H:%M:%S"), content = files, } - local f = io.open(luaname,'w') - if f then - input.report("saving " .. dataname .. " in", luaname) - f:write(input.serialize(data)) - f:close() - input.report("compiling " .. dataname .. " to", lucname) - if not utils.lua.compile(luaname,lucname) then - input.report("compiling failed for " .. dataname .. ", deleting file " .. lucname) + local ok = io.savedata(luaname,input.serialize(data)) + if ok then + input.report("%s saved in %s",dataname,luaname) + if utils.lua.compile(luaname,lucname,false,true) then -- no cleanup but strip + input.report("%s compiled to %s",dataname,lucname) + else + input.report("compiling failed for %s, deleting file %s",dataname,lucname) os.remove(lucname) end else - input.report("unable to save " .. dataname .. " in " .. name..input.luasuffix) + input.report("unable to save %s in %s (access error)",dataname,luaname) end end end -function input.aux.load_data(instance,pathname,dataname,filename,makename) -- untested without cache overload +function input.aux.load_data(pathname,dataname,filename,makename) -- untested without cache overload + local instance = input.instance filename = ((not filename or (filename == "")) and dataname) or filename filename = (makename and makename(dataname,filename)) or file.join(pathname,filename) - local blob = loadfile(filename .. input.lucsuffix) or loadfile(filename .. input.luasuffix) + local blob = loadfile(filename .. ".luc") or loadfile(filename .. ".lua") if blob then local data = blob() if data and data.content and data.type == dataname and data.version == input.cacheversion then - input.report("loading",dataname,"for",pathname,"from",filename) + input.report("loading %s for %s from %s",dataname,pathname,filename) instance[dataname][pathname] = data.content else - input.report("skipping",dataname,"for",pathname,"from",filename) + input.report("skipping %s for %s from %s",dataname,pathname,filename) instance[dataname][pathname] = { } instance.loaderror = true end else - input.report("skipping",dataname,"for",pathname,"from",filename) + input.report("skipping %s for %s from %s",dataname,pathname,filename) end end @@ -5712,13 +5902,14 @@ end -- TEXMFBOGUS = 'effe checken of dit werkt', -- } -function input.aux.load_texmfcnf(instance,dataname,pathname) +function input.aux.load_texmfcnf(dataname,pathname) + local instance = input.instance local filename = file.join(pathname,input.luaname) local blob = loadfile(filename) if blob then local data = blob() if data then - input.report("loading","configuration file",filename) + input.report("loading configuration file %s",filename) if true then -- flatten to variable.progname local t = { } @@ -5738,172 +5929,168 @@ function input.aux.load_texmfcnf(instanc instance[dataname][pathname] = data end else - input.report("skipping","configuration file",filename) + input.report("skipping configuration file %s",filename) instance[dataname][pathname] = { } instance.loaderror = true end else - input.report("skipping","configuration file",filename) + input.report("skipping configuration file %s",filename) end end -function input.aux.load_configuration(instance,dname,lname) - input.aux.load_data(instance,dname,'configuration',lname and file.basename(lname)) +function input.aux.load_configuration(dname,lname) + input.aux.load_data(dname,'configuration',lname and file.basename(lname)) end -function input.aux.load_files(instance,tag) - input.aux.load_data(instance,tag,'files') +function input.aux.load_files(tag) + input.aux.load_data(tag,'files') end -function input.resetconfig(instance) +function input.resetconfig() + input.identify_own() + local instance = input.instance instance.configuration, instance.setup, instance.order, instance.loaderror = { }, { }, { }, false end -function input.loadnewconfig(instance) +function input.loadnewconfig() + local instance = input.instance for _, cnf in ipairs(instance.luafiles) do local dname = file.dirname(cnf) - input.aux.load_texmfcnf(instance,'setup',dname) + input.aux.load_texmfcnf('setup',dname) instance.order[#instance.order+1] = instance.setup[dname] if instance.loaderror then break end end end -function input.loadoldconfig(instance) +function input.loadoldconfig() + local instance = input.instance if not instance.renewcache then for _, cnf in ipairs(instance.cnffiles) do local dname = file.dirname(cnf) - input.aux.load_configuration(instance,dname) + input.aux.load_configuration(dname) instance.order[#instance.order+1] = instance.configuration[dname] if instance.loaderror then break end end end - input.joinconfig(instance) + input.joinconfig() end -function input.expand_variables(instance) - instance.expansions = { } ---~ instance.environment['SELFAUTOPARENT'] = instance.environment['SELFAUTOPARENT'] or instance.rootpath - if instance.engine ~= "" then instance.environment['engine'] = instance.engine end - if instance.progname ~= "" then instance.environment['progname'] = instance.progname end - for k,v in pairs(instance.environment) do +function input.expand_variables() + local instance = input.instance + local expansions, environment, variables = { }, instance.environment, instance.variables + local env = input.env + instance.expansions = expansions + if instance.engine ~= "" then environment['engine'] = instance.engine end + if instance.progname ~= "" then environment['progname'] = instance.progname end + for k,v in pairs(environment) do local a, b = k:match("^(%a+)%_(.*)%s*$") if a and b then - instance.expansions[a..'.'..b] = v + expansions[a..'.'..b] = v else - instance.expansions[k] = v + expansions[k] = v end end - for k,v in pairs(instance.environment) do -- move environment to expansions - if not instance.expansions[k] then instance.expansions[k] = v end + for k,v in pairs(environment) do -- move environment to expansions + if not expansions[k] then expansions[k] = v end end - for k,v in pairs(instance.variables) do -- move variables to expansions - if not instance.expansions[k] then instance.expansions[k] = v end + for k,v in pairs(variables) do -- move variables to expansions + if not expansions[k] then expansions[k] = v end end while true do local busy = false - for k,v in pairs(instance.expansions) do + for k,v in pairs(expansions) do local s, n = v:gsub("%$([%a%d%_%-]+)", function(a) busy = true - return instance.expansions[a] or input.env(instance,a) + return expansions[a] or env(a) end) local s, m = s:gsub("%$%{([%a%d%_%-]+)%}", function(a) busy = true - return instance.expansions[a] or input.env(instance,a) + return expansions[a] or env(a) end) if n > 0 or m > 0 then - instance.expansions[k]= s + expansions[k]= s end end if not busy then break end end - local homedir = - instance.environment[(os.type == "windows" and 'USERPROFILE') or 'HOME'] or '~' - for k,v in pairs(instance.expansions) do - v = v:gsub("^~", homedir) - instance.expansions[k] = v:gsub("\\", '/') + for k,v in pairs(expansions) do + expansions[k] = v:gsub("\\", '/') end end -function input.aux.expand_vars(instance,lst) -- simple vars +function input.aux.expand_vars(lst) -- simple vars + local instance = input.instance + local variables, env = instance.variables, input.env for k,v in pairs(lst) do lst[k] = v:gsub("%$([%a%d%_%-]+)", function(a) - return instance.variables[a] or input.env(instance,a) + return variables[a] or env(a) end) end end -function input.aux.expanded_var(instance,var) -- simple vars +function input.aux.expanded_var(var) -- simple vars + local instance = input.instance return var:gsub("%$([%a%d%_%-]+)", function(a) - return instance.variables[a] or input.env(instance,a) + return instance.variables[a] or input.env(a) end) end -function input.aux.entry(instance,entries,name) +function input.aux.entry(entries,name) if name and (name ~= "") then + local instance = input.instance name = name:gsub('%$','') local result = entries[name..'.'..instance.progname] or entries[name] if result then return result else - result = input.env(instance,name) + result = input.env(name) if result then instance.variables[name] = result - input.expand_variables(instance) + input.expand_variables() return instance.expansions[name] or "" end end end return "" end -function input.variable(instance,name) - return input.aux.entry(instance,instance.variables,name) +function input.variable(name) + return input.aux.entry(input.instance.variables,name) end -function input.expansion(instance,name) - return input.aux.entry(instance,instance.expansions,name) +function input.expansion(name) + return input.aux.entry(input.instance.expansions,name) end -function input.aux.is_entry(instance,entries,name) +function input.aux.is_entry(entries,name) if name and name ~= "" then name = name:gsub('%$','') - return (entries[name..'.'..instance.progname] or entries[name]) ~= nil + return (entries[name..'.'..input.instance.progname] or entries[name]) ~= nil else return false end end -function input.is_variable(instance,name) - return input.aux.is_entry(instance,instance.variables,name) -end -function input.is_expansion(instance,name) - return input.aux.is_entry(instance,instance.expansions,name) +function input.is_variable(name) + return input.aux.is_entry(input.instance.variables,name) end -function input.simplified_list(str) - if type(str) == 'table' then - return str -- troubles ; ipv , in texmf - elseif str == '' then - return { } - else - local t = { } - for _,v in ipairs(string.splitchr(str:gsub("^\{(.+)\}$","%1"),",")) do - t[#t+1] = (v:gsub("^[%!]*(.+)[%/\\]*$","%1")) - end - return t - end +function input.is_expansion(name) + return input.aux.is_entry(input.instance.expansions,name) end -function input.unexpanded_path_list(instance,str) - local pth = input.variable(instance,str) +function input.unexpanded_path_list(str) + local pth = input.variable(str) local lst = input.split_path(pth) - return input.aux.expanded_path(instance,lst) + return input.aux.expanded_path(lst) end -function input.unexpanded_path(instance,str) - return file.join_path(input.unexpanded_path_list(instance,str)) + +function input.unexpanded_path(str) + return file.join_path(input.unexpanded_path_list(str)) end do local done = { } - function input.reset_extra_path(instance) + function input.reset_extra_path() + local instance = input.instance local ep = instance.extra_paths if not ep then ep, done = { }, { } @@ -5913,7 +6100,8 @@ do end end - function input.register_extra_path(instance,paths,subpaths) + function input.register_extra_path(paths,subpaths) + local instance = input.instance local ep = instance.extra_paths or { } local n = #ep if paths and paths ~= "" then @@ -5958,7 +6146,8 @@ do end -function input.expanded_path_list(instance,str) +function input.expanded_path_list(str) + local instance = input.instance local function made_list(list) local ep = instance.extra_paths if not ep or #ep == 0 then @@ -5999,39 +6188,41 @@ function input.expanded_path_list(instan -- engine+progname hash str = str:gsub("%$","") if not instance.lists[str] then -- cached - local lst = made_list(input.split_path(input.expansion(instance,str))) - instance.lists[str] = input.aux.expanded_path(instance,lst) + local lst = made_list(input.split_path(input.expansion(str))) + instance.lists[str] = input.aux.expanded_path(lst) end return instance.lists[str] else - local lst = input.split_path(input.expansion(instance,str)) - return made_list(input.aux.expanded_path(instance,lst)) + local lst = input.split_path(input.expansion(str)) + return made_list(input.aux.expanded_path(lst)) end end -function input.expand_path(instance,str) - return file.join_path(input.expanded_path_list(instance,str)) + +function input.clean_path_list(str) + local t = input.expanded_path_list(str) + if t then + for i=1,#t do + t[i] = file.collapse_path(input.clean_path(t[i])) + end + end + return t end ---~ function input.first_writable_path(instance,name) ---~ for _,v in pairs(input.expanded_path_list(instance,name)) do ---~ if file.is_writable(file.join(v,'luatex-cache.tmp')) then ---~ return v ---~ end ---~ end ---~ return "." ---~ end +function input.expand_path(str) + return file.join_path(input.expanded_path_list(str)) +end -function input.expanded_path_list_from_var(instance,str) -- brrr +function input.expanded_path_list_from_var(str) -- brrr local tmp = input.var_of_format_or_suffix(str:gsub("%$","")) if tmp ~= "" then - return input.expanded_path_list(instance,str) + return input.expanded_path_list(str) else - return input.expanded_path_list(instance,tmp) + return input.expanded_path_list(tmp) end end -function input.expand_path_from_var(instance,str) - return file.join_path(input.expanded_path_list_from_var(instance,str)) +function input.expand_path_from_var(str) + return file.join_path(input.expanded_path_list_from_var(str)) end function input.format_of_var(str) @@ -6061,9 +6252,9 @@ function input.var_of_format_or_suffix(s return '' end -function input.expand_braces(instance,str) -- output variable and brace expansion of STRING - local ori = input.variable(instance,str) - local pth = input.aux.expanded_path(instance,input.split_path(ori)) +function input.expand_braces(str) -- output variable and brace expansion of STRING + local ori = input.variable(str) + local pth = input.aux.expanded_path(input.split_path(ori)) return file.join_path(pth) end @@ -6078,6 +6269,7 @@ end -- {a,b,c/{p,q,r}/d/{x,y,z}//} -- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}} -- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}} +-- {$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c} -- this one is better and faster, but it took me a while to realize -- that this kind of replacement is cleaner than messy parsing and @@ -6086,19 +6278,19 @@ end -- work that well; the parsing is ok, but dealing with the resulting -- table is a pain because we need to work inside-out recursively --- get rid of piecewise here, just a gmatch is ok - function input.aux.splitpathexpr(str, t, validate) -- no need for optimization, only called a few times, we can use lpeg for the sub t = t or { } - local concat = table.concat + str = str:gsub(",}",",@}") + str = str:gsub("{,","{@,") + -- str = "@" .. str .. "@" while true do local done = false while true do local ok = false - str = str:gsub("([^{},]+){([^{}]-)}", function(a,b) + str = str:gsub("([^{},]+){([^{}]+)}", function(a,b) local t = { } - b:piecewise(",", function(s) t[#t+1] = a .. s end) + for s in b:gmatch("[^,]+") do t[#t+1] = a .. s end ok, done = true, true return "{" .. concat(t,",") .. "}" end) @@ -6106,9 +6298,9 @@ function input.aux.splitpathexpr(str, t, end while true do local ok = false - str = str:gsub("{([^{}]-)}([^{},]+)", function(a,b) + str = str:gsub("{([^{}]+)}([^{},]+)", function(a,b) local t = { } - a:piecewise(",", function(s) t[#t+1] = s .. b end) + for s in a:gmatch("[^,]+") do t[#t+1] = s .. b end ok, done = true, true return "{" .. concat(t,",") .. "}" end) @@ -6116,50 +6308,41 @@ function input.aux.splitpathexpr(str, t, end while true do local ok = false - str = str:gsub("([,{]){([^{}]+)}([,}])", function(a,b,c) + str = str:gsub("{([^{}]+)}{([^{}]+)}", function(a,b) + local t = { } + for sa in a:gmatch("[^,]+") do + for sb in b:gmatch("[^,]+") do + t[#t+1] = sa .. sb + end + end ok, done = true, true - return a .. b .. c + return "{" .. concat(t,",") .. "}" end) if not ok then break end end - if not done then break end - end - while true do - local ok = false - str = str:gsub("{([^{}]-)}{([^{}]-)}", function(a,b) - local t = { } - a:piecewise(",", function(sa) - b:piecewise(",", function(sb) - t[#t+1] = sa .. sb - end) - end) - ok = true - return "{" .. concat(t,",") .. "}" - end) - if not ok then break end - end - while true do - local ok = false - str = str:gsub("{([^{}]-)}", function(a) - ok = true - return a + str = str:gsub("({[^{}]*){([^{}]+)}([^{}]*})", function(a,b,c) + done = true + return a .. b.. c end) - if not ok then break end + if not done then break end end + str = str:gsub("[{}]", "") + str = str:gsub("@","") if validate then - str:piecewise(",", function(s) + for s in str:gmatch("[^,]+") do s = validate(s) if s then t[#t+1] = s end - end) + end else - str:piecewise(",", function(s) + for s in str:gmatch("[^,]+") do t[#t+1] = s - end) + end end return t end -function input.aux.expanded_path(instance,pathlist) -- maybe not a list, just a path +function input.aux.expanded_path(pathlist) -- maybe not a list, just a path + local instance = input.instance -- a previous version fed back into pathlist local newlist, ok = { }, false for _,v in ipairs(pathlist) do @@ -6191,17 +6374,16 @@ input.is_readable = { } function input.aux.is_readable(readable, name) if input.trace > 2 then if readable then - input.logger("+ readable", name) + input.logger("+ readable: %s",name) else - input.logger("- readable", name) + input.logger("- readable: %s", name) end end return readable end function input.is_readable.file(name) - -- return input.aux.is_readable(file.is_readable(name), name) - return input.aux.is_readable(input.aux.is_file(name), name) + return input.aux.is_readable(lfs.isfile(name), name) end input.is_readable.tex = input.is_readable.file @@ -6209,12 +6391,13 @@ input.is_readable.tex = input.is_readabl -- name -- name/name -function input.aux.collect_files(instance,names) +function input.aux.collect_files(names) + local instance = input.instance local filelist = { } for _, fname in pairs(names) do if fname then if input.trace > 2 then - input.logger("? blobpath asked",fname) + input.logger("? blobpath asked: %s",fname) end local bname = file.basename(fname) local dname = file.dirname(fname) @@ -6228,7 +6411,7 @@ function input.aux.collect_files(instanc local files = blobpath and instance.files[blobpath] if files then if input.trace > 2 then - input.logger('? blobpath do',blobpath .. " (" .. bname ..")") + input.logger('? blobpath do: %s (%s)',blobpath,bname) end local blobfile = files[bname] if not blobfile then @@ -6261,7 +6444,7 @@ function input.aux.collect_files(instanc end end elseif input.trace > 1 then - input.logger('! blobpath no',blobpath .. " (" .. bname ..")" ) + input.logger('! blobpath no: %s (%s)',blobpath,bname) end end end @@ -6316,15 +6499,17 @@ do end -function input.aux.register_in_trees(instance,name) +function input.aux.register_in_trees(name) if not name:find("^%.") then + local instance = input.instance instance.foundintrees[name] = (instance.foundintrees[name] or 0) + 1 -- maybe only one end end -- split the next one up, better for jit -function input.aux.find_file(instance,filename) -- todo : plugin (scanners, checkers etc) +function input.aux.find_file(filename) -- todo : plugin (scanners, checkers etc) + local instance = input.instance local result = { } local stamp = nil filename = input.normalize_name(filename) -- elsewhere @@ -6333,16 +6518,22 @@ function input.aux.find_file(instance,fi if instance.remember then stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format if instance.found[stamp] then - input.logger('! remembered', filename) + if input.trace > 0 then + input.logger('! remembered: %s',filename) + end return instance.found[stamp] end end if filename:find('%*') then - input.logger('! wildcard', filename) - result = input.find_wildcard_files(instance,filename) + if input.trace > 0 then + input.logger('! wildcard: %s', filename) + end + result = input.find_wildcard_files(filename) elseif input.aux.qualified_path(filename) then if input.is_readable.file(filename) then - input.logger('! qualified', filename) + if input.trace > 0 then + input.logger('! qualified: %s', filename) + end result = { filename } else local forcedname, ok = "", false @@ -6350,22 +6541,26 @@ function input.aux.find_file(instance,fi if instance.format == "" then forcedname = filename .. ".tex" if input.is_readable.file(forcedname) then - input.logger('! no suffix, forcing standard filetype tex') + if input.trace > 0 then + input.logger('! no suffix, forcing standard filetype: tex') + end result, ok = { forcedname }, true end else for _, s in pairs(input.suffixes_of_format(instance.format)) do forcedname = filename .. "." .. s if input.is_readable.file(forcedname) then - input.logger('! no suffix, forcing format filetype', s) + if input.trace > 0 then + input.logger('! no suffix, forcing format filetype: %s', s) + end result, ok = { forcedname }, true break end end end end - if not ok then - input.logger('? qualified', filename) + if not ok and input.trace > 0 then + input.logger('? qualified: %s', filename) end end else @@ -6383,10 +6578,14 @@ function input.aux.find_file(instance,fi local forcedname = filename .. '.tex' wantedfiles[#wantedfiles+1] = forcedname filetype = input.format_of_suffix(forcedname) - input.logger('! forcing filetype',filetype) + if input.trace > 0 then + input.logger('! forcing filetype: %s',filetype) + end else filetype = input.format_of_suffix(filename) - input.logger('! using suffix based filetype',filetype) + if input.trace > 0 then + input.logger('! using suffix based filetype: %s',filetype) + end end else if ext == "" then @@ -6395,16 +6594,18 @@ function input.aux.find_file(instance,fi end end filetype = instance.format - input.logger('! using given filetype',filetype) + if input.trace > 0 then + input.logger('! using given filetype: %s',filetype) + end end local typespec = input.variable_of_format(filetype) - local pathlist = input.expanded_path_list(instance,typespec) + local pathlist = input.expanded_path_list(typespec) if not pathlist or #pathlist == 0 then -- no pathlist, access check only / todo == wildcard if input.trace > 2 then - input.logger('? filename',filename) - input.logger('? filetype',filetype or '?') - input.logger('? wanted files',table.concat(wantedfiles," | ")) + input.logger('? filename: %s',filename) + input.logger('? filetype: %s',filetype or '?') + input.logger('? wanted files: %s',concat(wantedfiles," | ")) end for _, fname in pairs(wantedfiles) do if fname and input.is_readable.file(fname) then @@ -6414,7 +6615,7 @@ function input.aux.find_file(instance,fi end end -- this is actually 'other text files' or 'any' or 'whatever' - local filelist = input.aux.collect_files(instance,wantedfiles) + local filelist = input.aux.collect_files(wantedfiles) local fl = filelist and filelist[1] if fl then filename = fl[3] @@ -6423,12 +6624,12 @@ function input.aux.find_file(instance,fi end else -- list search - local filelist = input.aux.collect_files(instance,wantedfiles) + local filelist = input.aux.collect_files(wantedfiles) local doscan, recurse if input.trace > 2 then - input.logger('? filename',filename) - -- if pathlist then input.logger('? path list',table.concat(pathlist," | ")) end - -- if filelist then input.logger('? file list',table.concat(filelist," | ")) end + input.logger('? filename: %s',filename) + -- if pathlist then input.logger('? path list: %s',concat(pathlist," | ")) end + -- if filelist then input.logger('? file list: %s',concat(filelist," | ")) end end -- a bit messy ... esp the doscan setting here for _, path in pairs(pathlist) do @@ -6449,11 +6650,11 @@ function input.aux.find_file(instance,fi if f:find(expr) then -- input.debug('T',' '..f) if input.trace > 2 then - input.logger('= found in hash',f) + input.logger('= found in hash: %s',f) end --- todo, test for readable result[#result+1] = fl[3] - input.aux.register_in_trees(instance,f) -- for tracing used files + input.aux.register_in_trees(f) -- for tracing used files done = true if not instance.allresults then break end else @@ -6467,12 +6668,12 @@ function input.aux.find_file(instance,fi local pname = pathname:gsub("%.%*$",'') if not pname:find("%*") then local ppname = pname:gsub("/+$","") - if input.aux.can_be_dir(instance,ppname) then + if input.aux.can_be_dir(ppname) then for _, w in pairs(wantedfiles) do local fname = file.join(ppname,w) if input.is_readable.file(fname) then if input.trace > 2 then - input.logger('= found by scanning',fname) + input.logger('= found by scanning: %s',fname) end result[#result+1] = fname done = true @@ -6501,40 +6702,29 @@ function input.aux.find_file(instance,fi return result end -input.aux._find_file_ = input.aux.find_file +input.aux._find_file_ = input.aux.find_file -- frozen variant -function input.aux.find_file(instance,filename) -- maybe make a lowres cache too - local result = input.aux._find_file_(instance,filename) +function input.aux.find_file(filename) -- maybe make a lowres cache too + local result = input.aux._find_file_(filename) if #result == 0 then local lowered = filename:lower() if filename ~= lowered then - return input.aux._find_file_(instance,lowered) + return input.aux._find_file_(lowered) end end return result end -if lfs and lfs.isfile then - input.aux.is_file = lfs.isfile -- to be done: use this -else - input.aux.is_file = file.is_readable -end - -if lfs and lfs.isdir then - function input.aux.can_be_dir(instance,name) - if not instance.fakepaths[name] then - if lfs.isdir(name) then - instance.fakepaths[name] = 1 -- directory - else - instance.fakepaths[name] = 2 -- no directory - end +function input.aux.can_be_dir(name) + local instance = input.instance + if not instance.fakepaths[name] then + if lfs.isdir(name) then + instance.fakepaths[name] = 1 -- directory + else + instance.fakepaths[name] = 2 -- no directory end - return (instance.fakepaths[name] == 1) - end -else - function input.aux.can_be_dir() - return true end + return (instance.fakepaths[name] == 1) end if not input.concatinators then input.concatinators = { } end @@ -6542,7 +6732,8 @@ if not input.concatinators then input.c input.concatinators.tex = file.join input.concatinators.file = input.concatinators.tex -function input.find_files(instance,filename,filetype,mustexist) +function input.find_files(filename,filetype,mustexist) + local instance = input.instance if type(mustexist) == boolean then -- all set elseif type(filetype) == 'boolean' then @@ -6551,16 +6742,17 @@ function input.find_files(instance,filen filetype, mustexist = nil, false end instance.format = filetype or '' - local t = input.aux.find_file(instance,filename,true) + local t = input.aux.find_file(filename,true) instance.format = '' return t end -function input.find_file(instance,filename,filetype,mustexist) - return (input.find_files(instance,filename,filetype,mustexist)[1] or "") +function input.find_file(filename,filetype,mustexist) + return (input.find_files(filename,filetype,mustexist)[1] or "") end -function input.find_given_files(instance,filename) +function input.find_given_files(filename) + local instance = input.instance local bname, result = file.basename(filename), { } for k, hash in ipairs(instance.hashes) do local files = instance.files[hash.tag] @@ -6588,11 +6780,12 @@ function input.find_given_files(instance return result end -function input.find_given_file(instance,filename) - return (input.find_given_files(instance,filename)[1] or "") +function input.find_given_file(filename) + return (input.find_given_files(filename)[1] or "") end -function input.find_wildcard_files(instance,filename) -- todo: remap: +function input.find_wildcard_files(filename) -- todo: remap: + local instance = input.instance local result = { } local bname, dname = file.basename(filename), file.dirname(filename) local path = dname:gsub("^*/","") @@ -6645,16 +6838,19 @@ function input.find_wildcard_files(insta if done and not allresults then break end end end + -- we can consider also searching the paths not in the database, but then + -- we end up with a messy search (all // in all path specs) return result end -function input.find_wildcard_file(instance,filename) - return (input.find_wildcard_files(instance,filename)[1] or "") +function input.find_wildcard_file(filename) + return (input.find_wildcard_files(filename)[1] or "") end -- main user functions -function input.save_used_files_in_trees(instance, filename,jobname) +function input.save_used_files_in_trees(filename,jobname) + local instance = input.instance if not filename then filename = 'luatex.jlg' end local f = io.open(filename,'w') if f then @@ -6664,7 +6860,7 @@ function input.save_used_files_in_trees( f:write("\t<rl:name>" .. jobname .. "</rl:name>\n") end f:write("\t<rl:files>\n") - for _,v in pairs(table.sortedkeys(instance.foundintrees)) do + for _,v in pairs(sorted(instance.foundintrees)) do -- ipairs f:write("\t\t<rl:file n='" .. instance.foundintrees[v] .. "'>" .. v .. "</rl:file>\n") end f:write("\t</rl:files>\n") @@ -6673,24 +6869,24 @@ function input.save_used_files_in_trees( end end -function input.automount(instance) +function input.automount() -- implemented later end -function input.load(instance) - input.starttiming(instance) - input.resetconfig(instance) - input.identify_cnf(instance) - input.load_lua(instance) - input.expand_variables(instance) - input.load_cnf(instance) - input.expand_variables(instance) - input.load_hash(instance) - input.automount(instance) - input.stoptiming(instance) +function input.load() + input.starttiming(input.instance) + input.resetconfig() + input.identify_cnf() + input.load_lua() + input.expand_variables() + input.load_cnf() + input.expand_variables() + input.load_hash() + input.automount() + input.stoptiming(input.instance) end -function input.for_files(instance, command, files, filetype, mustexist) +function input.for_files(command, files, filetype, mustexist) if files and #files > 0 then local function report(str) if input.verbose then @@ -6703,7 +6899,7 @@ function input.for_files(instance, comma report('') end for _, file in pairs(files) do - local result = command(instance,file,filetype,mustexist) + local result = command(file,filetype,mustexist) if type(result) == 'string' then report(result) else @@ -6717,14 +6913,11 @@ end -- strtab -function input.var_value(instance,str) -- output the value of variable $STRING. - return input.variable(instance,str) -end -function input.expand_var(instance,str) -- output variable expansion of STRING. - return input.expansion(instance,str) -end -function input.show_path(instance,str) -- output search path for file type NAME - return file.join_path(input.expanded_path_list(instance,input.format_of_var(str))) +input.var_value = input.variable -- output the value of variable $STRING. +input.expand_var = input.expansion -- output variable expansion of STRING. + +function input.show_path(str) -- output search path for file type NAME + return file.join_path(input.expanded_path_list(input.format_of_var(str))) end -- input.find_file(filename) @@ -6773,56 +6966,58 @@ function table.sequenced(t,sep) -- temp for k, v in pairs(t) do s[#s+1] = k .. "=" .. v end - return table.concat(s, sep or " | ") + return concat(s, sep or " | ") end -function input.methodhandler(what, instance, filename, filetype) -- ... +function input.methodhandler(what, filename, filetype) -- ... local specification = (type(filename) == "string" and input.splitmethod(filename)) or filename -- no or { }, let it bomb local scheme = specification.scheme if input[what][scheme] then - input.logger('= handler',specification.original .." -> " .. what .. " -> " .. table.sequenced(specification)) - return input[what][scheme](instance,filename,filetype) -- todo: specification + if input.trace > 0 then + input.logger('= handler: %s -> %s -> %s',specification.original,what,table.sequenced(specification)) + end + return input[what][scheme](filename,filetype) -- todo: specification else - return input[what].tex(instance,filename,filetype) -- todo: specification + return input[what].tex(filename,filetype) -- todo: specification end end -- also inside next test? -function input.findtexfile(instance, filename, filetype) - return input.methodhandler('finders',instance, input.normalize_name(filename), filetype) +function input.findtexfile(filename, filetype) + return input.methodhandler('finders',input.normalize_name(filename), filetype) end -function input.opentexfile(instance,filename) - return input.methodhandler('openers',instance, input.normalize_name(filename)) +function input.opentexfile(filename) + return input.methodhandler('openers',input.normalize_name(filename)) end -function input.findbinfile(instance, filename, filetype) - return input.methodhandler('finders',instance, input.normalize_name(filename), filetype) +function input.findbinfile(filename, filetype) + return input.methodhandler('finders',input.normalize_name(filename), filetype) end -function input.openbinfile(instance,filename) - return input.methodhandler('loaders',instance, input.normalize_name(filename)) +function input.openbinfile(filename) + return input.methodhandler('loaders',input.normalize_name(filename)) end -function input.loadbinfile(instance, filename, filetype) - local fname = input.findbinfile(instance, input.normalize_name(filename), filetype) +function input.loadbinfile(filename, filetype) + local fname = input.findbinfile(input.normalize_name(filename), filetype) if fname and fname ~= "" then - return input.openbinfile(instance,fname) + return input.openbinfile(fname) else return unpack(input.loaders.notfound) end end -function input.texdatablob(instance, filename, filetype) - local ok, data, size = input.loadbinfile(instance, filename, filetype) +function input.texdatablob(filename, filetype) + local ok, data, size = input.loadbinfile(filename, filetype) return data or "" end input.loadtexfile = input.texdatablob -function input.openfile(filename) -- brrr texmf.instance here / todo ! ! ! ! ! - local fullname = input.findtexfile(texmf.instance, filename) +function input.openfile(filename) + local fullname = input.findtexfile(filename) if fullname and (fullname ~= "") then - return input.opentexfile(texmf.instance, fullname) + return input.opentexfile(fullname) else return nil end @@ -6868,16 +7063,18 @@ end -- beware: i need to check where we still need a / on windows: function input.clean_path(str) ---~ return (((str:gsub("\\","/")):gsub("^!+","")):gsub("//+","//")) if str then - return ((str:gsub("\\","/")):gsub("^!+","")) + str = str:gsub("\\","/") + str = str:gsub("^!+","") + str = str:gsub("^~",input.homedir) + return str else return nil end end function input.do_with_path(name,func) - for _, v in pairs(input.expanded_path_list(instance,name)) do + for _, v in pairs(input.expanded_path_list(name)) do func("^"..input.clean_path(v)) end end @@ -6886,7 +7083,8 @@ function input.do_with_var(name,func) func(input.aux.expanded_var(name)) end -function input.with_files(instance,pattern,handle) +function input.with_files(pattern,handle) + local instance = input.instance for _, hash in ipairs(instance.hashes) do local blobpath = hash.tag local blobtype = hash.type @@ -6913,37 +7111,22 @@ function input.with_files(instance,patte end end ---~ function input.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix ---~ newname = file.addsuffix(newname,"lua") ---~ local newscript = input.clean_path(input.find_file(instance, newname)) ---~ local oldscript = input.clean_path(oldname) ---~ input.report("old script", oldscript) ---~ input.report("new script", newscript) ---~ if oldscript ~= newscript and (oldscript:find(file.removesuffix(newname).."$") or oldscript:find(newname.."$")) then ---~ local newdata = io.loaddata(newscript) ---~ if newdata then ---~ input.report("old script content replaced by new content") ---~ io.savedata(oldscript,newdata) ---~ end ---~ end ---~ end - -function input.update_script(instance,oldname,newname) -- oldname -> own.name, not per se a suffix +function input.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix local scriptpath = "scripts/context/lua" newname = file.addsuffix(newname,"lua") local oldscript = input.clean_path(oldname) - input.report("to be replaced old script", oldscript) - local newscripts = input.find_files(instance, newname) or { } + input.report("to be replaced old script %s", oldscript) + local newscripts = input.find_files(newname) or { } if #newscripts == 0 then input.report("unable to locate new script") else for _, newscript in ipairs(newscripts) do newscript = input.clean_path(newscript) - input.report("checking new script", newscript) + input.report("checking new script %s", newscript) if oldscript == newscript then input.report("old and new script are the same") elseif not newscript:find(scriptpath) then - input.report("new script should come from",scriptpath) + input.report("new script should come from %s",scriptpath) elseif not (oldscript:find(file.removesuffix(newname).."$") or oldscript:find(newname.."$")) then input.report("invalid new script name") else @@ -6971,10 +7154,10 @@ do local resolvers = { } - resolvers.environment = function(instance,str) + resolvers.environment = function(str) return input.clean_path(os.getenv(str) or os.getenv(str:upper()) or os.getenv(str:lower()) or "") end - resolvers.relative = function(instance,str,n) + resolvers.relative = function(str,n) if io.exists(str) then -- nothing elseif io.exists("./" .. str) then @@ -6992,16 +7175,16 @@ do end return input.clean_path(str) end - resolvers.locate = function(instance,str) - local fullname = input.find_given_file(instance,str) or "" + resolvers.locate = function(str) + local fullname = input.find_given_file(str) or "" return input.clean_path((fullname ~= "" and fullname) or str) end - resolvers.filename = function(instance,str) - local fullname = input.find_given_file(instance,str) or "" + resolvers.filename = function(str) + local fullname = input.find_given_file(str) or "" return input.clean_path(file.basename((fullname ~= "" and fullname) or str)) end - resolvers.pathname = function(instance,str) - local fullname = input.find_given_file(instance,str) or "" + resolvers.pathname = function(str) + local fullname = input.find_given_file(str) or "" return input.clean_path(file.dirname((fullname ~= "" and fullname) or str)) end @@ -7013,15 +7196,15 @@ do resolvers.file = resolvers.filename resolvers.path = resolvers.pathname - local function resolve(instance,str) + local function resolve(str) if type(str) == "table" then for k, v in pairs(str) do - str[k] = resolve(instance,v) or v + str[k] = resolve(v) or v end elseif str and str ~= "" then - str = str:gsub("([a-z]+):([^ ]+)", function(method,target) + str = str:gsub("([a-z]+):([^ \"\']*)", function(method,target) if resolvers[method] then - return resolvers[method](instance,target) + return resolvers[method](target) else return method .. ":" .. target end @@ -7030,10 +7213,185 @@ do return str end + if os.uname then + for k, v in pairs(os.uname()) do + if not resolvers[k] then + resolvers[k] = function() return v end + end + end + end + input.resolve = resolve end +function input.boolean_variable(str,default) + local b = input.expansion(str) + if b == "" then + return default + else + b = toboolean(b) + return (b == nil and default) or b + end +end + + +if not modules then modules = { } end modules ['luat-log'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +--[[ldx-- +<p>This is a prelude to a more extensive logging module. For the sake +of parsing log files, in addition to the standard logging we will +provide an <l n='xml'/> structured file. Actually, any logging that +is hooked into callbacks will be \XML\ by default.</p> +--ldx]]-- + +-- input.logger -> special tracing, driven by log level (only input) +-- input.report -> goes to terminal, depends on verbose, has banner +-- logs.report -> module specific tracing and reporting, no banner but class + + +input = input or { } +logs = logs or { } + +--[[ldx-- +<p>This looks pretty ugly but we need to speed things up a bit.</p> +--ldx]]-- + +logs.levels = { + ['error'] = 1, + ['warning'] = 2, + ['info'] = 3, + ['debug'] = 4 +} + +logs.functions = { + 'report', 'start', 'stop', 'push', 'pop', 'line', 'direct' +} + +logs.callbacks = { + 'start_page_number', + 'stop_page_number', + 'report_output_pages', + 'report_output_log' +} + +logs.tracers = { +} + +logs.xml = logs.xml or { } +logs.tex = logs.tex or { } + +logs.level = 0 + +local write_nl, write, format = texio.write_nl or print, texio.write or io.write, string.format + +if texlua then + write_nl = print + write = io.write +end + +function logs.xml.report(category,fmt,...) -- new + write_nl(format("<r category='%s'>%s</r>",category,format(fmt,...))) +end +function logs.xml.line(fmt,...) -- new + write_nl(format("<r>%s</r>",format(fmt,...))) +end + +function logs.xml.start() if logs.level > 0 then tw("<%s>" ) end end +function logs.xml.stop () if logs.level > 0 then tw("</%s>") end end +function logs.xml.push () if logs.level > 0 then tw("<!-- ") end end +function logs.xml.pop () if logs.level > 0 then tw(" -->" ) end end + +function logs.tex.report(category,fmt,...) -- new + -- write_nl(format("%s | %s",category,format(fmt,...))) -- arg to format can be tex comment so .. . + write_nl(category .. " | " .. format(fmt,...)) +end +function logs.tex.line(fmt,...) -- new + write_nl(format(fmt,...)) +end + +function logs.set_level(level) + logs.level = logs.levels[level] or level +end + +function logs.set_method(method) + for _, v in pairs(logs.functions) do + logs[v] = logs[method][v] or function() end + end + if callback and input[method] then + for _, cb in pairs(logs.callbacks) do + callback.register(cb, input[method][cb]) + end + end +end + +function logs.xml.start_page_number() + write_nl(format("<p real='%s' page='%s' sub='%s'", tex.count[0], tex.count[1], tex.count[2])) +end + +function logs.xml.stop_page_number() + write("/>") + write_nl("") +end + +function logs.xml.report_output_pages(p,b) + write_nl(format("<v k='pages' v='%s'/>", p)) + write_nl(format("<v k='bytes' v='%s'/>", b)) + write_nl("") +end + +function logs.xml.report_output_log() +end + +function input.logger(...) -- assumes test for input.trace > n + if input.trace > 0 then + logs.report(...) + end +end + +function input.report(fmt,...) + if input.verbose then + logs.report(input.banner or "report",format(fmt,...)) + end +end + +function input.reportlines(str) -- todo: <lines></lines> + for line in str:gmatch("(.-)[\n\r]") do + logs.report(input.banner or "report",line) + end +end + +input.moreinfo = [[ +more information about ConTeXt and the tools that come with it can be found at: + +maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context +webpage : http://www.pragma-ade.nl / http://tex.aanhet.net +wiki : http://contextgarden.net +]] + +function input.help(banner,message) + if not input.verbose then + input.verbose = true + -- input.report(banner,"\n") + end + input.report(banner,"\n") + input.report("") + input.reportlines(message) + if input.moreinfo and input.moreinfo ~= "" then + input.report("") + input.reportlines(input.moreinfo) + end +end + +logs.set_level('error') +logs.set_method('tex') + if not modules then modules = { } end modules ['luat-tmp'] = { version = 1.001, @@ -7059,63 +7417,82 @@ being written at the same time is small. luatools with a recache feature.</p> --ldx]]-- +local format = string.format + caches = caches or { } dir = dir or { } texmf = texmf or { } -caches.path = caches.path or nil -caches.base = caches.base or "luatex-cache" -caches.more = caches.more or "context" -caches.direct = false -- true is faster but may need huge amounts of memory -caches.trace = false -caches.tree = false -caches.paths = caches.paths or nil -caches.force = false - -input.usecache = not toboolean(os.getenv("TEXMFSHARECACHE") or "false",true) -- true - -function caches.temp(instance) - local function checkpath(cachepath) - if not cachepath or cachepath == "" then - return nil - elseif lfs.attributes(cachepath,"mode") == "directory" then -- lfs.isdir(cachepath) then - return cachepath - elseif caches.force or io.ask(string.format("Should I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then - dir.mkdirs(cachepath) - return (lfs.attributes(cachepath,"mode") == "directory") and cachepath - else - return nil +caches.path = caches.path or nil +caches.base = caches.base or "luatex-cache" +caches.more = caches.more or "context" +caches.direct = false -- true is faster but may need huge amounts of memory +caches.trace = false +caches.tree = false +caches.paths = caches.paths or nil +caches.force = false +caches.defaults = { "TEXMFCACHE", "TMPDIR", "TEMPDIR", "TMP", "TEMP", "HOME", "HOMEPATH" } + +function caches.temp() + local cachepath = nil + local function check(list,isenv) + if not cachepath then + for _, v in ipairs(list) do + cachepath = (isenv and (os.env[v] or "")) or v or "" + if cachepath == "" then + -- next + else + cachepath = input.clean_path(cachepath) + if lfs.isdir(cachepath) and file.iswritable(cachepath) then -- lfs.attributes(cachepath,"mode") == "directory" + break + elseif caches.force or io.ask(format("\nShould I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then + dir.mkdirs(cachepath) + if lfs.isdir(cachepath) and file.iswritable(cachepath) then + break + end + end + end + cachepath = nil + end end end - local cachepath = input.expanded_path_list(instance,"TEXMFCACHE") - cachepath = cachepath and #cachepath > 0 and checkpath(cachepath[1]) - if not cachepath then - cachepath = os.getenv("TEXMFCACHE") or os.getenv("HOME") or os.getenv("HOMEPATH") or os.getenv("TMP") or os.getenv("TEMP") or os.getenv("TMPDIR") or nil - cachepath = checkpath(cachepath) - end + check(input.clean_path_list("TEXMFCACHE") or { }) + check(caches.defaults,true) if not cachepath then - print("\nfatal error: there is no valid cache path defined\n") + print("\nfatal error: there is no valid (writable) cache path defined\n") os.exit() - elseif lfs.attributes(cachepath,"mode") ~= "directory" then - print(string.format("\nfatal error: cache path %s is not a directory\n",cachepath)) + elseif not lfs.isdir(cachepath) then -- lfs.attributes(cachepath,"mode") ~= "directory" + print(format("\nfatal error: cache path %s is not a directory\n",cachepath)) os.exit() end - function caches.temp(instance) + cachepath = input.normalize_name(cachepath) + function caches.temp() return cachepath end return cachepath end -function caches.configpath(instance) - return table.concat(instance.cnffiles,";") +function caches.configpath() + return table.concat(input.instance.cnffiles,";") end function caches.hashed(tree) return md5.hex((tree:lower()):gsub("[\\\/]+","/")) end -function caches.treehash(instance) - local tree = caches.configpath(instance) +--~ tracing: + +--~ function caches.hashed(tree) +--~ tree = (tree:lower()):gsub("[\\\/]+","/") +--~ local hash = md5.hex(tree) +--~ if input.verbose then -- temp message +--~ input.report("hashing %s => %s",tree,hash) +--~ end +--~ return hash +--~ end + +function caches.treehash() + local tree = caches.configpath() if not tree or tree == "" then return false else @@ -7123,14 +7500,14 @@ function caches.treehash(instance) end end -function caches.setpath(instance,...) +function caches.setpath(...) if not caches.path then if not caches.path then - caches.path = caches.temp(instance) + caches.path = caches.temp() end caches.path = input.clean_path(caches.path) -- to be sure if lfs then - caches.tree = caches.tree or caches.treehash(instance) + caches.tree = caches.tree or caches.treehash() if caches.tree then caches.path = dir.mkdirs(caches.path,caches.base,caches.more,caches.tree) else @@ -7150,9 +7527,9 @@ function caches.setpath(instance,...) return caches.path end -function caches.definepath(instance,category,subcategory) +function caches.definepath(category,subcategory) return function() - return caches.setpath(instance,category,subcategory) + return caches.setpath(category,subcategory) end end @@ -7175,26 +7552,28 @@ function caches.is_writable(filepath,fil return file.is_writable(tmaname) end -function caches.savedata(filepath,filename,data,raw) -- raw needed for file cache +function caches.savedata(filepath,filename,data,raw) local tmaname, tmcname = caches.setluanames(filepath,filename) local reduce, simplify = true, true if raw then reduce, simplify = false, false end if caches.direct then - file.savedata(tmaname, table.serialize(data,'return',true,true)) + file.savedata(tmaname, table.serialize(data,'return',true,true,false)) -- no hex else - table.tofile (tmaname, data,'return',true,true) -- maybe not the last true + table.tofile(tmaname, data,'return',true,true,false) -- maybe not the last true end - utils.lua.compile(tmaname, tmcname) + local cleanup = input.boolean_variable("PURGECACHE", false) + local strip = input.boolean_variable("LUACSTRIP", true) + utils.lua.compile(tmaname, tmcname, cleanup, strip) end -- here we use the cache for format loading (texconfig.[formatname|jobname]) --~ if tex and texconfig and texconfig.formatname and texconfig.formatname == "" then -if tex and texconfig and (not texconfig.formatname or texconfig.formatname == "") and texmf.instance then +if tex and texconfig and (not texconfig.formatname or texconfig.formatname == "") and input and input.instance then if not texconfig.luaname then texconfig.luaname = "cont-en.lua" end -- or luc - texconfig.formatname = caches.setpath(texmf.instance,"formats") .. "/" .. texconfig.luaname:gsub("%.lu.$",".fmt") + texconfig.formatname = caches.setpath("formats") .. "/" .. texconfig.luaname:gsub("%.lu.$",".fmt") end --[[ldx-- @@ -7217,7 +7596,7 @@ do -- local report local function report(container,tag,name) if caches.trace or containers.trace or container.trace then - logs.report(string.format("%s cache",container.subcategory),string.format("%s: %s",tag,name or 'invalid')) + logs.report(format("%s cache",container.subcategory),"%s: %s",tag,name or 'invalid') end end @@ -7242,7 +7621,7 @@ do -- local report enabled = enabled, version = version or 1.000, trace = false, - path = caches.setpath(texmf.instance,category,subcategory), + path = caches.setpath(category,subcategory), } c[subcategory] = s end @@ -7307,13 +7686,16 @@ end -- reimplement the saver. local save_data = input.aux.save_data +local load_data = input.aux.load_data -input.cachepath = nil +input.cachepath = nil -- public, for tracing +input.usecache = true -- public, for tracing -function input.aux.save_data(instance, dataname, check) - input.cachepath = input.cachepath or caches.definepath(instance,"trees") - save_data(instance, dataname, check, function(cachename,dataname) +function input.aux.save_data(dataname, check) + save_data(dataname, check, function(cachename,dataname) + input.usecache = not toboolean(input.expansion("CACHEINTDS") or "false",true) if input.usecache then + input.cachepath = input.cachepath or caches.definepath("trees") return file.join(input.cachepath(),caches.hashed(cachename)) else return file.join(cachename,dataname) @@ -7321,12 +7703,11 @@ function input.aux.save_data(instance, d end) end -local load_data = input.aux.load_data - -function input.aux.load_data(instance,pathname,dataname,filename) - input.cachepath = input.cachepath or caches.definepath(instance,"trees") - load_data(instance,pathname,dataname,filename,function(dataname,filename) +function input.aux.load_data(pathname,dataname,filename) + load_data(pathname,dataname,filename,function(dataname,filename) + input.usecache = not toboolean(input.expansion("CACHEINTDS") or "false",true) if input.usecache then + input.cachepath = input.cachepath or caches.definepath("trees") return file.join(input.cachepath(),caches.hashed(pathname)) else if not filename or (filename == "") then @@ -7341,13 +7722,13 @@ end input.automounted = input.automounted or { } -function input.automount(instance,usecache) - local mountpaths = input.simplified_list(input.expansion(instance,'TEXMFMOUNT')) +function input.automount(usecache) + local mountpaths = input.clean_path_list(input.expansion('TEXMFMOUNT')) if table.is_empty(mountpaths) and usecache then - mountpaths = { caches.setpath(instance,"mount") } + mountpaths = { caches.setpath("mount") } end if not table.is_empty(mountpaths) then - input.starttiming(instance) + input.starttiming(input.instance) for k, root in pairs(mountpaths) do local f = io.open(root.."/url.tmi") if f then @@ -7356,16 +7737,16 @@ function input.automount(instance,usecac if line:find("^[%%#%-]") then -- or %W -- skip elseif line:find("^zip://") then - input.report("mounting",line) + input.report("mounting %s",line) table.insert(input.automounted,line) - input.usezipfile(instance,line) + input.usezipfile(line) end end end f:close() end end - input.stoptiming(instance) + input.stoptiming(input.instance) end end @@ -7376,7 +7757,7 @@ input.storage.data = { } input.storage.min = 0 -- 500 input.storage.max = input.storage.min - 1 input.storage.trace = false -- true -input.storage.done = 0 +input.storage.done = input.storage.done or 0 input.storage.evaluators = { } -- (evaluate,message,names) @@ -7414,17 +7795,17 @@ function input.storage.dump() else name = str end - initialize = string.format("%s %s = %s or {} ", initialize, name, name) + initialize = format("%s %s = %s or {} ", initialize, name, name) end if evaluate then finalize = "input.storage.evaluate(" .. name .. ")" end input.storage.max = input.storage.max + 1 if input.storage.trace then - logs.report('storage',string.format('saving %s in slot %s',message,input.storage.max)) + logs.report('storage','saving %s in slot %s',message,input.storage.max) code = initialize .. - string.format("logs.report('storage','restoring %s from slot %s') ",message,input.storage.max) .. + format("logs.report('storage','restoring %s from slot %s') ",message,input.storage.max) .. table.serialize(original,name) .. finalize else @@ -7434,6 +7815,8 @@ function input.storage.dump() end end +-- we also need to count at generation time (nicer for message) + if lua.bytecode then -- from 0 upwards local i = input.storage.min while lua.bytecode[i] do @@ -7460,6 +7843,11 @@ provide an <l n='xml'/> structured file. is hooked into callbacks will be \XML\ by default.</p> --ldx]]-- +-- input.logger -> special tracing, driven by log level (only input) +-- input.report -> goes to terminal, depends on verbose, has banner +-- logs.report -> module specific tracing and reporting, no banner but class + + input = input or { } logs = logs or { } @@ -7475,8 +7863,7 @@ logs.levels = { } logs.functions = { - 'error', 'warning', 'info', 'debug', 'report', - 'start', 'stop', 'push', 'pop' + 'report', 'start', 'stop', 'push', 'pop', 'line', 'direct' } logs.callbacks = { @@ -7486,89 +7873,112 @@ logs.callbacks = { 'report_output_log' } +logs.tracers = { +} + logs.xml = logs.xml or { } logs.tex = logs.tex or { } logs.level = 0 -do - local write_nl, write, format = texio.write_nl or print, texio.write or io.write, string.format +local write_nl, write, format = texio.write_nl or print, texio.write or io.write, string.format - if texlua then - write_nl = print - write = io.write - end +if texlua then + write_nl = print + write = io.write +end - function logs.xml.debug(category,str) - if logs.level > 3 then write_nl(format("<d category='%s'>%s</d>",category,str)) end - end - function logs.xml.info(category,str) - if logs.level > 2 then write_nl(format("<i category='%s'>%s</i>",category,str)) end - end - function logs.xml.warning(category,str) - if logs.level > 1 then write_nl(format("<w category='%s'>%s</w>",category,str)) end - end - function logs.xml.error(category,str) - if logs.level > 0 then write_nl(format("<e category='%s'>%s</e>",category,str)) end - end - function logs.xml.report(category,str) - write_nl(format("<r category='%s'>%s</r>",category,str)) - end +function logs.xml.report(category,fmt,...) -- new + write_nl(format("<r category='%s'>%s</r>",category,format(fmt,...))) +end +function logs.xml.line(fmt,...) -- new + write_nl(format("<r>%s</r>",format(fmt,...))) +end - function logs.xml.start() if logs.level > 0 then tw("<%s>" ) end end - function logs.xml.stop () if logs.level > 0 then tw("</%s>") end end - function logs.xml.push () if logs.level > 0 then tw("<!-- ") end end - function logs.xml.pop () if logs.level > 0 then tw(" -->" ) end end +function logs.xml.start() if logs.level > 0 then tw("<%s>" ) end end +function logs.xml.stop () if logs.level > 0 then tw("</%s>") end end +function logs.xml.push () if logs.level > 0 then tw("<!-- ") end end +function logs.xml.pop () if logs.level > 0 then tw(" -->" ) end end - function logs.tex.debug(category,str) - if logs.level > 3 then write_nl(format("debug >> %s: %s" ,category,str)) end - end - function logs.tex.info(category,str) - if logs.level > 2 then write_nl(format("info >> %s: %s" ,category,str)) end - end - function logs.tex.warning(category,str) - if logs.level > 1 then write_nl(format("warning >> %s: %s",category,str)) end - end - function logs.tex.error(category,str) - if logs.level > 0 then write_nl(format("error >> %s: %s" ,category,str)) end - end - function logs.tex.report(category,str) - write_nl(format("report >> %s: %s" ,category,str)) - end +function logs.tex.report(category,fmt,...) -- new + -- write_nl(format("%s | %s",category,format(fmt,...))) -- arg to format can be tex comment so .. . + write_nl(category .. " | " .. format(fmt,...)) +end +function logs.tex.line(fmt,...) -- new + write_nl(format(fmt,...)) +end - function logs.set_level(level) - logs.level = logs.levels[level] or level - end +function logs.set_level(level) + logs.level = logs.levels[level] or level +end - function logs.set_method(method) - for _, v in pairs(logs.functions) do - logs[v] = logs[method][v] or function() end - end - if callback and input[method] then - for _, cb in pairs(logs.callbacks) do - callback.register(cb, input[method][cb]) - end +function logs.set_method(method) + for _, v in pairs(logs.functions) do + logs[v] = logs[method][v] or function() end + end + if callback and input[method] then + for _, cb in pairs(logs.callbacks) do + callback.register(cb, input[method][cb]) end end +end - function logs.xml.start_page_number() - write_nl(format("<p real='%s' page='%s' sub='%s'", tex.count[0], tex.count[1], tex.count[2])) - end +function logs.xml.start_page_number() + write_nl(format("<p real='%s' page='%s' sub='%s'", tex.count[0], tex.count[1], tex.count[2])) +end + +function logs.xml.stop_page_number() + write("/>") + write_nl("") +end + +function logs.xml.report_output_pages(p,b) + write_nl(format("<v k='pages' v='%s'/>", p)) + write_nl(format("<v k='bytes' v='%s'/>", b)) + write_nl("") +end + +function logs.xml.report_output_log() +end - function logs.xml.stop_page_number() - write("/>") - write_nl("") +function input.logger(...) -- assumes test for input.trace > n + if input.trace > 0 then + logs.report(...) end +end - function logs.xml.report_output_pages(p,b) - write_nl(format("<v k='pages' v='%s'/>", p)) - write_nl(format("<v k='bytes' v='%s'/>", b)) - write_nl("") +function input.report(fmt,...) + if input.verbose then + logs.report(input.banner or "report",format(fmt,...)) end +end - function logs.xml.report_output_log() +function input.reportlines(str) -- todo: <lines></lines> + for line in str:gmatch("(.-)[\n\r]") do + logs.report(input.banner or "report",line) end +end +input.moreinfo = [[ +more information about ConTeXt and the tools that come with it can be found at: + +maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context +webpage : http://www.pragma-ade.nl / http://tex.aanhet.net +wiki : http://contextgarden.net +]] + +function input.help(banner,message) + if not input.verbose then + input.verbose = true + -- input.report(banner,"\n") + end + input.report(banner,"\n") + input.report("") + input.reportlines(message) + if input.moreinfo and input.moreinfo ~= "" then + input.report("") + input.reportlines(input.moreinfo) + end end logs.set_level('error') @@ -7761,7 +8171,7 @@ end -- end library merge -own = { } +own = { } -- not local own.libs = { -- todo: check which ones are really needed 'l-string.lua', @@ -7775,12 +8185,14 @@ own.libs = { -- todo: check which ones a 'l-file.lua', 'l-dir.lua', 'l-boolean.lua', + 'l-math.lua', 'l-xml.lua', -- 'l-unicode.lua', 'l-utils.lua', -- 'l-tex.lua', 'luat-lib.lua', 'luat-inp.lua', + 'luat-log.lua', -- 'luat-zip.lua', -- 'luat-tex.lua', -- 'luat-kps.lua', @@ -7810,7 +8222,7 @@ table.insert(own.list,own.path.."/../../ table.insert(own.list,own.path.."/mtx") table.insert(own.list,own.path.."/../sources") -function locate_libs() +local function locate_libs() for _, lib in pairs(own.libs) do for _, pth in pairs(own.list) do local filename = string.gsub(pth .. "/" .. lib,"\\","/") @@ -7837,37 +8249,14 @@ if not input then os.exit() end -instance = input.reset() -input.verbose = environment.argument("verbose") or false -input.banner = 'MtxRun | ' +input.instance = input.reset() +input.banner = 'MtxRun' utils.report = input.report -instance.engine = environment.argument("engine") or 'luatex' -instance.progname = environment.argument("progname") or 'context' -instance.lsrmode = environment.argument("lsr") or false +local instance = input.instance --- use os.env or environment when available ---~ function input.check_environment(tree) ---~ input.report('') ---~ os.setenv('TMP', os.getenv('TMP') or os.getenv('TEMP') or os.getenv('TMPDIR') or os.getenv('HOME')) ---~ if os.platform == 'linux' then ---~ os.setenv('TEXOS', os.getenv('TEXOS') or 'texmf-linux') ---~ elseif os.platform == 'windows' then ---~ os.setenv('TEXOS', os.getenv('TEXOS') or 'texmf-windows') ---~ elseif os.platform == 'macosx' then ---~ os.setenv('TEXOS', os.getenv('TEXOS') or 'texmf-macosx') ---~ end ---~ os.setenv('TEXOS', string.gsub(string.gsub(os.getenv('TEXOS'),"^[\\\/]*", ''),"[\\\/]*$", '')) ---~ os.setenv('TEXPATH', string.gsub(tree,"\/+$",'')) ---~ os.setenv('TEXMFOS', os.getenv('TEXPATH') .. "/" .. os.getenv('TEXOS')) ---~ input.report('') ---~ input.report("preset : TEXPATH => " .. os.getenv('TEXPATH')) ---~ input.report("preset : TEXOS => " .. os.getenv('TEXOS')) ---~ input.report("preset : TEXMFOS => " .. os.getenv('TEXMFOS')) ---~ input.report("preset : TMP => " .. os.getenv('TMP')) ---~ input.report('') ---~ end +-- use os.env or environment when available function input.check_environment(tree) input.report('') @@ -7876,10 +8265,10 @@ function input.check_environment(tree) os.setenv('TEXPATH', (tree or "tex"):gsub("\/+$",'')) os.setenv('TEXMFOS', os.getenv('TEXPATH') .. "/" .. os.getenv('TEXOS')) input.report('') - input.report("preset : TEXPATH => " .. os.getenv('TEXPATH')) - input.report("preset : TEXOS => " .. os.getenv('TEXOS')) - input.report("preset : TEXMFOS => " .. os.getenv('TEXMFOS')) - input.report("preset : TMP => " .. os.getenv('TMP')) + input.report("preset : TEXPATH => %s", os.getenv('TEXPATH')) + input.report("preset : TEXOS => %s", os.getenv('TEXOS')) + input.report("preset : TEXMFOS => %s", os.getenv('TEXMFOS')) + input.report("preset : TMP => %s", os.getenv('TMP')) input.report('') end @@ -7968,9 +8357,9 @@ function file.needs_updating(oldname,new end end -function file.mdchecksum(name) +function file.checksum(name) if md5 then - local data = io.loadall(name) + local data = io.loaddata(name) if data then return md5.HEXsum(data) end @@ -7979,24 +8368,18 @@ function file.mdchecksum(name) end function file.loadchecksum(name) - if md then - local data = io.loadall(name .. ".md5") - if data then - return string.gsub(md5.HEXsum(data),"%s$","") - end + if md5 then + local data = io.loaddata(name .. ".md5") + return data and data:gsub("%s","") end return nil end function file.savechecksum(name, checksum) - if not checksum then checksum = file.mdchecksum(name) end + if not checksum then checksum = file.checksum(name) end if checksum then - local f = io.open(name .. ".md5","w") - if f then - f:write(checksum) - f:close() - return checksum - end + io.savedata(name .. ".md5",checksum) + return checksum end return nil end @@ -8014,6 +8397,8 @@ function os.currentplatform(name, defaul local architecture = os.arch() if architecture:find("x86_64") then return "linux-64" + elseif architecture:find("ppc") then + return "linux-ppc" else return "linux" end @@ -8055,7 +8440,7 @@ input.runners.registered = { mptopdf = { 'mptopdf.pl', true }, pstopdf = { 'pstopdf.rb', true }, - examplex = { 'examplex.rb', false }, +-- examplex = { 'examplex.rb', false }, concheck = { 'concheck.rb', false }, runtools = { 'runtools.rb', true }, @@ -8076,8 +8461,8 @@ input.runners.registered = { if not messages then messages = { } end messages.help = [[ ---script run an mtx script ---execute run a script or program +--script run an mtx script (--noquotes) +--execute run a script or program (--noquotes) --resolve resolve prefixed arguments --ctxlua run internally (using preloaded libs) --locate locate given filename @@ -8100,46 +8485,46 @@ messages.help = [[ --progname=str format or backend --edit launch editor with found file ---launch (--all) launch files (assume os support) +--launch (--all) launch files like manuals, assumes os support --intern run script using built in libraries ]] -function input.runners.my_prepare_a(instance) - input.resetconfig(instance) - input.identify_cnf(instance) - input.load_lua(instance) - input.expand_variables(instance) - input.load_cnf(instance) - input.expand_variables(instance) -end - -function input.runners.my_prepare_b(instance) - input.runners.my_prepare_a(instance) - input.load_hash(instance) - input.automount(instance) +function input.runners.my_prepare_a() + input.resetconfig() + input.identify_cnf() + input.load_lua() + input.expand_variables() + input.load_cnf() + input.expand_variables() +end + +function input.runners.my_prepare_b() + input.runners.my_prepare_a() + input.load_hash() + input.automount() end -function input.runners.prepare(instance) +function input.runners.prepare() local checkname = environment.argument("ifchanged") if checkname and checkname ~= "" then local oldchecksum = file.loadchecksum(checkname) local newchecksum = file.checksum(checkname) if oldchecksum == newchecksum then - report("file '" .. checkname .. "' is unchanged") + input.report("file '%s' is unchanged",checkname) return "skip" else - report("file '" .. checkname .. "' is changed, processing started") + input.report("file '%s' is changed, processing started",checkname) end file.savechecksum(checkname) end local oldname, newname = string.split(environment.argument("iftouched") or "", ",") if oldname and newname and oldname ~= "" and newname ~= "" then if not file.needs_updating(oldname,newname) then - report("file '" .. oldname .. "' and '" .. newname .. "'have same age") + input.report("file '%s' and '%s' have same age",oldname,newname) return "skip" else - report("file '" .. newname .. "' is older than '" .. oldname .. "'") + input.report("file '%s' is older than '%s'",oldname,newname) end end local tree = environment.argument('tree') or "" @@ -8158,15 +8543,17 @@ function input.runners.prepare(instance) end local runpath = environment.argument("path") if runpath and not dir.chdir(runpath) then - input.report("unable to change to path '" .. runpath .. "'") + input.report("unable to change to path '%s'",runpath) return "error" end return "run" end -function input.runners.execute_script(instance,fullname,internal) +function input.runners.execute_script(fullname,internal) + local instance = input.instance + local noquote = environment.argument("noquotes") if fullname and fullname ~= "" then - local state = input.runners.prepare(instance) + local state = input.runners.prepare() if state == 'error' then return false elseif state == 'skip' then @@ -8190,16 +8577,16 @@ function input.runners.execute_script(in if suffix == "" then -- loop over known suffixes for _,s in pairs(input.runners.suffixes) do - result = input.find_file(instance, name .. "." .. s, 'texmfscripts') + result = input.find_file(name .. "." .. s, 'texmfscripts') if result ~= "" then break end end elseif input.runners.applications[suffix] then - result = input.find_file(instance, name, 'texmfscripts') + result = input.find_file(name, 'texmfscripts') else -- maybe look on path - result = input.find_file(instance, name, 'other text files') + result = input.find_file(name, 'other text files') end end if result and result ~= "" then @@ -8213,9 +8600,9 @@ function input.runners.execute_script(in result = binary .. " " .. result end local before, after = environment.split_arguments(fullname) - local command = result .. " " .. environment.reconstruct_commandline(after) + local command = result .. " " .. environment.reconstruct_commandline(after,noquote) input.report("") - input.report("executing: " .. command) + input.report("executing: %s",command) input.report("\n \n") io.flush() local code = os.exec(command) -- maybe spawn @@ -8227,9 +8614,10 @@ function input.runners.execute_script(in return false end -function input.runners.execute_program(instance,fullname) +function input.runners.execute_program(fullname) + local noquote = environment.argument("noquotes") if fullname and fullname ~= "" then - local state = input.runners.prepare(instance) + local state = input.runners.prepare() if state == 'error' then return false elseif state == 'skip' then @@ -8238,9 +8626,9 @@ function input.runners.execute_program(i local before, after = environment.split_arguments(fullname) environment.initialize_arguments(after) fullname = fullname:gsub("^bin:","") - local command = fullname .. " " .. environment.reconstruct_commandline(after) + local command = fullname .. " " .. (environment.reconstruct_commandline(after or "",noquote) or "") input.report("") - input.report("executing: " .. command) + input.report("executing: %s",command) input.report("\n \n") io.flush() local code = os.exec(command) -- (fullname,unpack(after)) does not work / maybe spawn @@ -8250,12 +8638,12 @@ function input.runners.execute_program(i return false end -function input.runners.handle_stubs(instance,create) +function input.runners.handle_stubs(create) local stubpath = environment.argument('stubpath') or '.' -- 'auto' no longer supported local windows = environment.argument('windows') or environment.argument('mswin') or false local unix = environment.argument('unix') or environment.argument('linux') or false if not windows and not unix then - if environment.platform == "unix" then + if os.platform == "unix" then unix = true else windows = true @@ -8270,41 +8658,41 @@ function input.runners.handle_stubs(inst local command = "luatex --luaonly mtxrun.lua " .. name if windows then io.savedata(base..".bat", {"@echo off", command.." %*"}, "\013\010") - input.report("windows stub for '" .. base .. "' created") + input.report("windows stub for '%s' created",base) end if unix then io.savedata(base, {"#!/bin/sh", command..' "$@"'}, "\010") - input.report("unix stub for '" .. base .. "' created") + input.report("unix stub for '%s' created",base) end else if windows and (os.remove(base..'.bat') or os.remove(base..'.cmd')) then - input.report("windows stub for '" .. base .. "' removed") + input.report("windows stub for '%s' removed", base) end if unix and (os.remove(base) or os.remove(base..'.sh')) then - input.report("unix stub for '" .. base .. "' removed") + input.report("unix stub for '%s' removed",base) end end end end end -function input.runners.resolve_string(instance,filename) +function input.runners.resolve_string(filename) if filename and filename ~= "" then - input.runners.report_location(instance,input.resolve(instance,filename)) + input.runners.report_location(input.resolve(filename)) end end -function input.runners.locate_file(instance,filename) +function input.runners.locate_file(filename) if filename and filename ~= "" then - input.runners.report_location(instance,input.find_given_file(instance,filename)) + input.runners.report_location(input.find_given_file(filename)) end end -function input.runners.locate_platform(instance) - input.runners.report_location(instance,os.currentplatform()) +function input.runners.locate_platform() + input.runners.report_location(os.currentplatform()) end -function input.runners.report_location(instance,result) +function input.runners.report_location(result) if input.verbose then input.report("") if result and result ~= "" then @@ -8317,11 +8705,17 @@ function input.runners.report_location(i end end -function input.runners.edit_script(instance,filename) - local editor = os.getenv("MTXRUN_EDITOR") or os.getenv("TEXMFSTART_EDITOR") or os.getenv("EDITOR") or 'scite' - local rest = input.resolve(instance,filename) +function input.runners.edit_script(filename) -- we assume that vim is present on most systems + local editor = os.getenv("MTXRUN_EDITOR") or os.getenv("TEXMFSTART_EDITOR") or os.getenv("EDITOR") or 'vim' + local rest = input.resolve(filename) if rest ~= "" then - os.launch(editor .. " " .. rest) + local command = editor .. " " .. rest + if input.verbose then + input.report("") + input.report("starting editor: %s",command) + input.report("\n \n") + end + os.launch(command) end end @@ -8364,7 +8758,8 @@ function input.launch(str) os.launch(str) end -function input.runners.launch_file(instance,filename) +function input.runners.launch_file(filename) + local instance = input.instance instance.allresults = true input.verbose = true local pattern = environment.arguments["pattern"] @@ -8374,25 +8769,30 @@ function input.runners.launch_file(insta if not pattern or pattern == "" then input.report("provide name or --pattern=") else - local t = input.find_files(instance,pattern) - -- local t = input.aux.find_file(instance,"*/" .. pattern,true) + local t = input.find_files(pattern) + if not t or #t == 0 then + t = input.aux.find_file("*/" .. pattern,true) + end + if not t or #t == 0 then + t = input.aux.find_file("*/" .. pattern .. "*",true) + end if t and #t > 0 then if environment.arguments["all"] then for _, v in pairs(t) do - input.report("launching", v) + input.report("launching %s", v) input.launch(v) end else - input.report("launching", t[1]) + input.report("launching %s", t[1]) input.launch(t[1]) end else - input.report("no match for", pattern) + input.report("no match for %s", pattern) end end end -function input.runners.execute_ctx_script(instance,filename,arguments) +function input.runners.find_mtx_script(filename) local function found(name) local path = file.dirname(name) if path and path ~= "" then @@ -8402,30 +8802,48 @@ function input.runners.execute_ctx_scrip return io.exists(fullname) and fullname end end - local suffix = "" - if not filename:find("%.lua$") then suffix = ".lua" end - local fullname = filename - -- just <filename> - fullname = filename .. suffix - fullname = input.find_file(instance,fullname) - -- mtx-<filename> - if not fullname or fullname == "" then - fullname = "mtx-" .. filename .. suffix - fullname = found(fullname) or input.find_file(instance,fullname) - end - -- mtx-<filename>s - if not fullname or fullname == "" then - fullname = "mtx-" .. filename .. "s" .. suffix - fullname = found(fullname) or input.find_file(instance,fullname) - end - -- mtx-<filename minus trailing s> - if not fullname or fullname == "" then - fullname = "mtx-" .. filename:gsub("s$","") .. suffix - fullname = found(fullname) or input.find_file(instance,fullname) + filename = file.addsuffix(filename,"lua") + local basename = file.stripsuffix(file.basename(filename)) + local suffix = file.extname(filename) + -- qualified path, raw name + local fullname = input.aux.qualified_path(filename) and io.exists(filename) and filename + if fullname and fullname ~= "" then + return fullname + end + -- current path, raw name + fullname = "./" .. filename + fullname = io.exists(fullname) and fullname + if fullname and fullname ~= "" then + return fullname + end + -- context namespace, mtx-<filename> + fullname = "mtx-" .. filename + fullname = found(fullname) or input.find_file(fullname) + if fullname and fullname ~= "" then + return fullname + end + -- context namespace, mtx-<filename>s + fullname = "mtx-" .. basename .. "s" .. "." .. suffix + fullname = found(fullname) or input.find_file(fullname) + if fullname and fullname ~= "" then + return fullname + end + -- context namespace, mtx-<filename minus trailing s> + fullname = "mtx-" .. basename:gsub("s$","") .. "." .. suffix + fullname = found(fullname) or input.find_file(fullname) + if fullname and fullname ~= "" then + return fullname end + -- context namespace, just <filename> + fullname = input.find_file(fullname) + return fullname +end + +function input.runners.execute_ctx_script(filename,arguments) + local fullname = input.runners.find_mtx_script(filename) -- that should do it if fullname and fullname ~= "" then - local state = input.runners.prepare(instance) + local state = input.runners.prepare() if state == 'error' then return false elseif state == 'skip' then @@ -8442,7 +8860,7 @@ function input.runners.execute_ctx_scrip end filename = environment.files[1] if input.verbose then - input.report("using script: " .. fullname) + input.report("using script: %s\n",fullname) end dofile(fullname) local savename = environment.arguments['save'] @@ -8455,20 +8873,19 @@ function input.runners.execute_ctx_scrip end else input.verbose = true - input.report("unknown script: " .. filename) + filename = file.addsuffix(filename,"lua") + if filename == "" then + input.report("unknown script, no name given") + elseif input.aux.qualified_path(filename) then + input.report("unknown script '%s'",filename) + else + input.report("unknown script '%s' or 'mtx-%s'",filename,filename) + end return false end end -input.report(banner,"\n") - -function input.help(banner,message) - if not input.verbose then - input.verbose = true - input.report(banner,"\n") - end - input.reportlines(message) -end +input.report("%s\n",banner) -- this is a bit dirty ... first we store the first filename and next we -- split the arguments so that we only see the ones meant for this script @@ -8478,12 +8895,14 @@ local filename = environment.files[1] or local ok = true local before, after = environment.split_arguments(filename) +environment.initialize_arguments(before) -input.runners.my_prepare_b(instance) -before = input.resolve(instance,before) -- experimental here -after = input.resolve(instance,after) -- experimental here +instance.engine = environment.argument("engine") or 'luatex' +instance.progname = environment.argument("progname") or 'context' +instance.lsrmode = environment.argument("lsr") or false +input.verbose = environment.argument("verbose") or false -environment.initialize_arguments(before) +input.runners.my_prepare_b() if environment.argument("selfmerge") then -- embed used libraries @@ -8493,60 +8912,52 @@ elseif environment.argument("selfclean") utils.merger.selfclean(own.name) elseif environment.argument("selfupdate") then input.verbose = true - input.update_script(instance,own.name,"mtxrun") + input.update_script(own.name,"mtxrun") elseif environment.argument("ctxlua") or environment.argument("internal") then -- run a script by loading it (using libs) - ok = input.runners.execute_script(instance,filename,true) + ok = input.runners.execute_script(filename,true) elseif environment.argument("script") then -- run a script by loading it (using libs), pass args - ok = input.runners.execute_ctx_script(instance,filename,after) + ok = input.runners.execute_ctx_script(filename,after) elseif environment.argument("execute") then -- execute script - ok = input.runners.execute_script(instance,filename) + ok = input.runners.execute_script(filename) elseif environment.argument("direct") then -- equals bin: - ok = input.runners.execute_program(instance,filename) + ok = input.runners.execute_program(filename) elseif environment.argument("edit") then -- edit file - input.runners.edit_script(instance,filename) + input.runners.edit_script(filename) elseif environment.argument("launch") then - input.runners.launch_file(instance,filename) + input.runners.launch_file(filename) elseif environment.argument("make") then -- make stubs - input.runners.handle_stubs(instance,true) + input.runners.handle_stubs(true) elseif environment.argument("remove") then -- remove stub - input.runners.handle_stubs(instance,false) + input.runners.handle_stubs(false) elseif environment.argument("resolve") then -- resolve string - input.runners.resolve_string(instance,filename) + input.runners.resolve_string(filename) elseif environment.argument("locate") then -- locate file - input.runners.locate_file(instance,filename) + input.runners.locate_file(filename) elseif environment.argument("platform")then -- locate platform - input.runners.locate_platform(instance) + input.runners.locate_platform() elseif environment.argument("help") or filename=='help' or filename == "" then input.help(banner,messages.help) -- execute script - if filename:find("^bin:") then - ok = input.runners.execute_program(instance,filename) - else - ok = input.runners.execute_script(instance,filename) - end +elseif filename:find("^bin:") then + ok = input.runners.execute_program(filename) +else + ok = input.runners.execute_script(filename) end ---~ if input.verbose then ---~ input.report("") ---~ input.report(string.format("runtime: %0.3f seconds",os.runtime())) ---~ end - ---~ if ok then ---~ input.report("exit code: 0") os.exit(0) ---~ else ---~ input.report("exit code: 1") os.exit(1) ---~ end - -if environment.platform == "unix" then +if os.platform == "unix" then io.write("\n") end + +if ok == false then ok = 1 elseif ok == true then ok = 0 end + +os.exit(ok) --- texmf-dist/tex/context/base/luat-inp.lua +++ texmf-dist/tex/context/base/luat-inp.lua 2008-10-30 14:05:43.990092457 +0100 @@ -1,18 +1,21 @@ --- filename : luat-inp.lua --- comment : companion to luat-lib.tex --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files - --- This lib is multi-purpose and can be loaded again later on so that --- additional functionality becomes available. We will split this --- module in components when we're done with prototyping. +if not modules then modules = { } end modules ['luat-inp'] = { + version = 1.001, + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files", + comment = "companion to luat-lib.tex", +} -- TODO: os.getenv -> os.env[] -- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller) -- TODO: check escaping in find etc, too much, too slow --- This is the first code I wrote for LuaTeX, so it needs some cleanup. +-- This lib is multi-purpose and can be loaded again later on so that +-- additional functionality becomes available. We will split this +-- module in components once we're done with prototyping. This is the +-- first code I wrote for LuaTeX, so it needs some cleanup. Before changing +-- something in this module one can best check with Taco or Hans first; there +-- is some nasty trickery going on that relates to traditional kpse support. -- To be considered: hash key lowercase, first entry in table filename -- (any case), rest paths (so no need for optimization). Or maybe a @@ -22,12 +25,6 @@ -- Beware, loading and saving is overloaded in luat-tmp! -if not versions then versions = { } end versions['luat-inp'] = 1.001 -if not environment then environment = { } end -if not file then file = { } end - -if environment.aleph_mode == nil then environment.aleph_mode = true end -- temp hack - if not input then input = { } end if not input.suffixes then input.suffixes = { } end if not input.formats then input.formats = { } end @@ -40,7 +37,7 @@ if not input.hashers then input.hashe if not input.generators then input.generators = { } end -- generate databases if not input.filters then input.filters = { } end -- conversion filters -local format = string.format +local format, concat, sortedkeys = string.format, table.concat, table.sortedkeys input.locators.notfound = { nil } input.hashers.notfound = { nil } @@ -53,8 +50,16 @@ input.debug = false input.cnfname = 'texmf.cnf' input.luaname = 'texmfcnf.lua' input.lsrname = 'ls-R' -input.luasuffix = '.tma' -input.lucsuffix = '.tmc' +input.homedir = os.env[os.platform == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~' + +--~ input.luasuffix = 'tma' +--~ input.lucsuffix = 'tmc' + +-- for the moment we have .local but this will disappear +input.cnfdefault = '{$SELFAUTOLOC,$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}' + +-- chances are low that the cnf file is in the bin path +input.cnfdefault = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}' -- we use a cleaned up list / format=any is a wildcard, as is *name @@ -90,7 +95,8 @@ input.suffixes['lua'] = { 'lua', 'luc', -- FONTFEATURES = .;$TEXMF/fonts/fea// -- FONTCIDMAPS = .;$TEXMF/fonts/cid// -function input.checkconfigdata(instance) -- not yet ok, no time for debugging now +function input.checkconfigdata() -- not yet ok, no time for debugging now + local instance = input.instance local function fix(varname,default) local proname = varname .. "." .. instance.progname or "crap" local p = instance.environment[proname] @@ -99,7 +105,15 @@ function input.checkconfigdata(instance) instance.variables[varname] = default -- or environment? end end - fix("LUAINPUTS" , ".;$TEXINPUTS;$TEXMFSCRIPTS") + local name = os.name + if name == "windows" then + fix("OSFONTDIR", "c:/windows/fonts//") + elseif name == "macosx" then + fix("OSFONTDIR", "$HOME/Library/Fonts//;/Library/Fonts//;/System/Library/Fonts//") + else + -- bad luck + end + fix("LUAINPUTS" , ".;$TEXINPUTS;$TEXMFSCRIPTS") -- no progname, hm fix("FONTFEATURES", ".;$TEXMF/fonts/fea//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") fix("FONTCIDMAPS" , ".;$TEXMF/fonts/cid//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") end @@ -126,14 +140,20 @@ input.formats ['sfd'] input.suffixes ['sfd'] = { 'sfd' } input.alternatives['subfont definition files'] = 'sfd' -function input.reset() +-- In practice we will work within one tds tree, but i want to keep +-- the option open to build tools that look at multiple trees, which is +-- why we keep the tree specific data in a table. We used to pass the +-- instance but for practical pusposes we now avoid this and use a +-- instance variable. + +function input.newinstance() local instance = { } instance.rootpath = '' instance.treepath = '' - instance.progname = environment.progname or 'context' - instance.engine = environment.engine or 'luatex' + instance.progname = 'context' + instance.engine = 'luatex' instance.format = '' instance.environment = { } instance.variables = { } @@ -157,6 +177,7 @@ function input.reset() instance.cachepath = nil instance.loaderror = false instance.smallcache = false + instance.sortdata = false instance.savelists = true instance.cleanuppaths = true instance.allresults = false @@ -172,23 +193,13 @@ function input.reset() instance.fakepaths = { } instance.lsrmode = false - if os.env then - -- store once, freeze and faster - for k,v in pairs(os.env) do - instance.environment[k] = input.bare_variable(v) - end - else - -- we will access os.env frequently - for k,v in pairs({'HOME','TEXMF','TEXMFCNF'}) do - local e = os.getenv(v) - if e then - -- input.report("setting",v,"to",input.bare_variable(e)) - instance.environment[v] = input.bare_variable(e) - end - end + -- store once, freeze and faster (once reset we can best use instance.environment) + + for k,v in pairs(os.env) do + instance.environment[k] = input.bare_variable(v) end - -- cross referencing + -- cross referencing, delayed because we can add suffixes for k, v in pairs(input.suffixes) do for _, vv in pairs(v) do @@ -202,68 +213,42 @@ function input.reset() end -function input.reset_hashes(instance) - instance.lists = { } - instance.found = { } -end - -function input.bare_variable(str) -- assumes str is a string - -- return string.gsub(string.gsub(string.gsub(str,"%s+$",""),'^"(.+)"$',"%1"),"^'(.+)'$","%1") - return (str:gsub("\s*([\"\']?)(.+)%1\s*", "%2")) -end +input.instance = input.instance or nil -if texio then - input.log = texio.write_nl -else - input.log = print +function input.reset() + input.instance = input.newinstance() + return input.instance end -function input.simple_logger(kind, name) - if name and name ~= "" then - if input.banner then - input.log(input.banner..kind..": "..name) - else - input.log("<<"..kind..": "..name..">>") - end - else - if input.banner then - input.log(input.banner..kind..": no name") - else - input.log("<<"..kind..": no name>>") - end - end +function input.reset_hashes() + input.instance.lists = { } + input.instance.found = { } end -function input.dummy_logger() +function input.bare_variable(str) -- assumes str is a string + -- return string.gsub(string.gsub(string.gsub(str,"%s+$",""),'^"(.+)"$',"%1"),"^'(.+)'$","%1") + return (str:gsub("\s*([\"\']?)(.+)%1\s*", "%2")) end function input.settrace(n) input.trace = tonumber(n or 0) if input.trace > 0 then - input.logger = input.simple_logger input.verbose = true - else - input.logger = function() end end end -function input.report(...) -- inefficient +input.log = (texio and texio.write_nl) or print + +function input.report(...) if input.verbose then - if input.banner then - input.log(input.banner .. table.concat({...},' ')) - elseif input.logmode() == 'xml' then - input.log("<t>"..table.concat({...},' ').."</t>") - else - input.log("<<"..table.concat({...},' ')..">>") - end + input.log("<<"..format(...)..">>") end end -function input.reportlines(str) - if type(str) == "string" then - str = str:split("\n") +function input.report(...) + if input.trace > 0 then -- extra test + input.log("<<"..format(...)..">>") end - for _,v in pairs(str) do input.report(v) end end input.settrace(tonumber(os.getenv("MTX.INPUT.TRACE") or os.getenv("MTX_INPUT_TRACE") or input.trace or 0)) @@ -274,27 +259,44 @@ input.settrace(tonumber(os.getenv("MTX.I do local clock = os.gettimeofday or os.clock + function input.hastimer(instance) + return instance and instance.starttime + end + function input.starttiming(instance) if instance then - instance.starttime = clock() - if not instance.loadtime then - instance.loadtime = 0 + local it = instance.timing + if not it then + it = 0 + end + if it == 0 then + instance.starttime = clock() + if not instance.loadtime then + instance.loadtime = 0 + end end + instance.timing = it + 1 end end function input.stoptiming(instance, report) if instance then - local starttime = instance.starttime - if starttime then - local stoptime = clock() - local loadtime = stoptime - starttime - instance.stoptime = stoptime - instance.loadtime = instance.loadtime + loadtime - if report then - input.report('load time', format("%0.3f",loadtime)) + local it = instance.timing + if it > 1 then + instance.timing = it - 1 + else + local starttime = instance.starttime + if starttime then + local stoptime = clock() + local loadtime = stoptime - starttime + instance.stoptime = stoptime + instance.loadtime = instance.loadtime + loadtime + if report then + input.report("load time %0.3f",loadtime) + end + instance.timing = 0 + return loadtime end - return loadtime end end return 0 @@ -308,18 +310,18 @@ end function input.report_loadtime(instance) if instance then - input.report('total load time', input.elapsedtime(instance)) + input.report('total load time %s', input.elapsedtime(instance)) end end input.loadtime = input.elapsedtime -function input.env(instance,key) - return instance.environment[key] or input.osenv(instance,key) +function input.env(key) + return input.instance.environment[key] or input.osenv(key) end -function input.osenv(instance,key) - local ie = instance.environment +function input.osenv(key) + local ie = input.instance.environment local value = ie[key] if value == nil then -- local e = os.getenv(key) @@ -337,81 +339,106 @@ end -- we follow a rather traditional approach: -- -- (1) texmf.cnf given in TEXMFCNF --- (2) texmf.cnf searched in TEXMF/web2c +-- (2) texmf.cnf searched in default variable -- --- for the moment we don't expect a configuration file in a zip +-- also we now follow the stupid route: if not set then just assume *one* +-- cnf file under texmf (i.e. distribution) -function input.identify_cnf(instance) - -- we no longer support treepath and rootpath (was handy for testing); - -- also we now follow the stupid route: if not set then just assume *one* - -- cnf file under texmf (i.e. distribution) - if #instance.cnffiles == 0 then - if input.env(instance,'TEXMFCNF') == "" then - local ownpath = environment.ownpath() or "." - if ownpath then - -- beware, this is tricky on my own system because at that location I do have - -- the raw tree that ends up in the zip; i.e. I cannot test this kind of mess - local function locate(filename,list) - local ownroot = input.normalize_name(file.join(ownpath,"../..")) - if not lfs.isdir(file.join(ownroot,"texmf")) then - ownroot = input.normalize_name(file.join(ownpath,"..")) - if not lfs.isdir(file.join(ownroot,"texmf")) then - input.verbose = true - input.report("error", "unable to identify cnf file") - return +input.ownpath = input.ownpath or nil +input.ownbin = input.ownbin or arg[-2] or arg[-1] or arg[0] or "luatex" +input.autoselfdir = true -- false may be handy for debugging + +function input.getownpath() + if not input.ownpath then + if input.autoselfdir and os.selfdir then + input.ownpath = os.selfdir + else + local binary = input.ownbin + if os.platform == "windows" then + binary = file.replacesuffix(binary,"exe") + end + for p in string.gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do + local b = file.join(p,binary) + if lfs.isfile(b) then + -- we assume that after changing to the path the currentdir function + -- resolves to the real location and use this side effect here; this + -- trick is needed because on the mac installations use symlinks in the + -- path instead of real locations + local olddir = lfs.currentdir() + if lfs.chdir(p) then + local pp = lfs.currentdir() + if input.verbose and p ~= pp then + input.report("following symlink %s to %s",p,pp) end - end - local texmfcnf = file.join(ownroot,"texmf-local/web2c",filename) -- for minimals and myself - if not lfs.isfile(texmfcnf) then - texmfcnf = file.join(ownroot,"texmf/web2c",filename) - if not lfs.isfile(texmfcnf) then - input.verbose = true - input.report("error", "unable to locate",filename) - return + input.ownpath = pp + lfs.chdir(olddir) + else + if input.verbose then + input.report("unable to check path %s",p) end + input.ownpath = p end - table.insert(list,texmfcnf) - local ie = instance.environment - if not ie['SELFAUTOPARENT'] then ie['SELFAUTOPARENT'] = ownroot end - if not ie['TEXMFCNF'] then ie['TEXMFCNF'] = file.dirname(texmfcnf) end - end - locate(input.luaname,instance.luafiles) - locate(input.cnfname,instance.cnffiles) - if #instance.luafiles == 0 and instance.cnffiles == 0 then - input.verbose = true - input.report("error", "unable to locate",filename) - os.exit() - end - -- here we also assume then TEXMF is set in the distribution, if this trickery is - -- used in the minimals, then users who don't use setuptex are on their own with - -- regards to extra trees - else - input.verbose = true - input.report("error", "unable to identify own path") - os.exit() + break + end end - else - local t = input.split_path(input.env(instance,'TEXMFCNF')) - t = input.aux.expanded_path(instance,t) - input.aux.expand_vars(instance,t) - local function locate(filename,list) - for _,v in ipairs(t) do - local texmfcnf = input.normalize_name(file.join(v,filename)) - if lfs.isfile(texmfcnf) then - table.insert(list,texmfcnf) - end + end + if not input.ownpath then input.ownpath = '.' end + end + return input.ownpath +end + +function input.identify_own() + local instance = input.instance + local ownpath = input.getownpath() or lfs.currentdir() + local ie = instance.environment + if ownpath then + if input.env('SELFAUTOLOC') == "" then os.env['SELFAUTOLOC'] = file.collapse_path(ownpath) end + if input.env('SELFAUTODIR') == "" then os.env['SELFAUTODIR'] = file.collapse_path(ownpath .. "/..") end + if input.env('SELFAUTOPARENT') == "" then os.env['SELFAUTOPARENT'] = file.collapse_path(ownpath .. "/../..") end + else + input.verbose = true + input.report("error: unable to locate ownpath") + os.exit() + end + if input.env('TEXMFCNF') == "" then os.env['TEXMFCNF'] = input.cnfdefault end + if input.env('TEXOS') == "" then os.env['TEXOS'] = input.env('SELFAUTODIR') end + if input.env('TEXROOT') == "" then os.env['TEXROOT'] = input.env('SELFAUTOPARENT') end + if input.verbose then + for _,v in ipairs({"SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF"}) do + input.report("variable %s set to %s",v,input.env(v) or "unknown") + end + end + function input.identify_own() end +end + +function input.identify_cnf() + local instance = input.instance + if #instance.cnffiles == 0 then + -- fallback + input.identify_own() + -- the real search + input.expand_variables() + local t = input.split_path(input.env('TEXMFCNF')) + t = input.aux.expanded_path(t) + input.aux.expand_vars(t) -- redundant + local function locate(filename,list) + for _,v in ipairs(t) do + local texmfcnf = input.normalize_name(file.join(v,filename)) + if lfs.isfile(texmfcnf) then + table.insert(list,texmfcnf) end end - locate(input.luaname,instance.luafiles) - locate(input.cnfname,instance.cnffiles) end + locate(input.luaname,instance.luafiles) + locate(input.cnfname,instance.cnffiles) end end -function input.load_cnf(instance) +function input.load_cnf() + local instance = input.instance local function loadoldconfigdata() for _, fname in ipairs(instance.cnffiles) do - input.aux.load_cnf(instance,fname) + input.aux.load_cnf(fname) end end -- instance.cnffiles contain complete names now ! @@ -426,27 +453,27 @@ function input.load_cnf(instance) instance.rootpath = file.dirname(instance.rootpath) end instance.rootpath = input.normalize_name(instance.rootpath) - instance.environment['SELFAUTOPARENT'] = instance.rootpath -- just to be sure if instance.lsrmode then loadoldconfigdata() elseif instance.diskcache and not instance.renewcache then - input.loadoldconfig(instance,instance.cnffiles) + input.loadoldconfig(instance.cnffiles) if instance.loaderror then loadoldconfigdata() - input.saveoldconfig(instance) + input.saveoldconfig() end else loadoldconfigdata() if instance.renewcache then - input.saveoldconfig(instance) + input.saveoldconfig() end end - input.aux.collapse_cnf_data(instance) + input.aux.collapse_cnf_data() end - input.checkconfigdata(instance) + input.checkconfigdata() end -function input.load_lua(instance) +function input.load_lua() + local instance = input.instance if #instance.luafiles == 0 then -- yet harmless else @@ -458,14 +485,14 @@ function input.load_lua(instance) instance.rootpath = file.dirname(instance.rootpath) end instance.rootpath = input.normalize_name(instance.rootpath) - instance.environment['SELFAUTOPARENT'] = instance.rootpath -- just to be sure - input.loadnewconfig(instance) - input.aux.collapse_cnf_data(instance) + input.loadnewconfig() + input.aux.collapse_cnf_data() end - input.checkconfigdata(instance) + input.checkconfigdata() end -function input.aux.collapse_cnf_data(instance) -- potential optmization: pass start index (setup and configuration are shared) +function input.aux.collapse_cnf_data() -- potential optimization: pass start index (setup and configuration are shared) + local instance = input.instance for _,c in ipairs(instance.order) do for k,v in pairs(c) do if not instance.variables[k] then @@ -480,21 +507,22 @@ function input.aux.collapse_cnf_data(ins end end -function input.aux.load_cnf(instance,fname) +function input.aux.load_cnf(fname) + local instance = input.instance fname = input.clean_path(fname) - local lname = fname:gsub("%.%a+$",input.luasuffix) + local lname = file.replacesuffix(fname,'lua') local f = io.open(lname) if f then -- this will go f:close() local dname = file.dirname(fname) if not instance.configuration[dname] then - input.aux.load_configuration(instance,dname,lname) + input.aux.load_configuration(dname,lname) instance.order[#instance.order+1] = instance.configuration[dname] end else f = io.open(fname) if f then - input.report("loading", fname) + input.report("loading %s", fname) local line, data, n, k, v local dname = file.dirname(fname) if not instance.configuration[dname] then @@ -526,227 +554,226 @@ function input.aux.load_cnf(instance,fna end f:close() else - input.report("skipping", fname) + input.report("skipping %s", fname) end end end -- database loading -function input.load_hash(instance) - input.locatelists(instance) +function input.load_hash() + local instance = input.instance + input.locatelists() if instance.lsrmode then - input.loadlists(instance) + input.loadlists() elseif instance.diskcache and not instance.renewcache then - input.loadfiles(instance) + input.loadfiles() if instance.loaderror then - input.loadlists(instance) - input.savefiles(instance) + input.loadlists() + input.savefiles() end else - input.loadlists(instance) + input.loadlists() if instance.renewcache then - input.savefiles(instance) + input.savefiles() end end end -function input.aux.append_hash(instance,type,tag,name) - input.logger("= hash append",tag) - table.insert(instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } ) +function input.aux.append_hash(type,tag,name) + if input.trace > 0 then + input.logger("= hash append: %s",tag) + end + table.insert(input.instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } ) end -function input.aux.prepend_hash(instance,type,tag,name) - input.logger("= hash prepend",tag) - table.insert(instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } ) +function input.aux.prepend_hash(type,tag,name) + if input.trace > 0 then + input.logger("= hash prepend: %s",tag) + end + table.insert(input.instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } ) end -function input.aux.extend_texmf_var(instance,specification) -- crap - if instance.environment['TEXMF'] then - input.report("extending environment variable TEXMF with", specification) - instance.environment['TEXMF'] = instance.environment['TEXMF']:gsub("^%{", function() - return "{" .. specification .. "," - end) - elseif instance.variables['TEXMF'] then - input.report("extending configuration variable TEXMF with", specification) - instance.variables['TEXMF'] = instance.variables['TEXMF']:gsub("^%{", function() - return "{" .. specification .. "," - end) +function input.aux.extend_texmf_var(specification) -- crap, we could better prepend the hash + local instance = input.instance +-- local t = input.expanded_path_list('TEXMF') -- full expansion + local t = input.split_path(input.env('TEXMF')) + table.insert(t,1,specification) + local newspec = table.join(t,";") + if instance.environment["TEXMF"] then + instance.environment["TEXMF"] = newspec + elseif instance.variables["TEXMF"] then + instance.variables["TEXMF"] = newspec else - input.report("setting configuration variable TEXMF to", specification) - instance.variables['TEXMF'] = "{" .. specification .. "}" - end - if instance.variables['TEXMF']:find("%,") and not instance.variables['TEXMF']:find("^%{") then - input.report("adding {} to complex TEXMF variable, best do that yourself") - instance.variables['TEXMF'] = "{" .. instance.variables['TEXMF'] .. "}" + -- weird end - input.expand_variables(instance) - input.reset_hashes(instance) + input.expand_variables() + input.reset_hashes() end -- locators -function input.locatelists(instance) - for _, path in pairs(input.simplified_list(input.expansion(instance,'TEXMF'))) do - path = file.collapse_path(path) - input.report("locating list of",path) - input.locatedatabase(instance,input.normalize_name(path)) +function input.locatelists() + local instance = input.instance + for _, path in pairs(input.clean_path_list('TEXMF')) do + input.report("locating list of %s",path) + input.locatedatabase(input.normalize_name(path)) end end -function input.locatedatabase(instance,specification) - return input.methodhandler('locators', instance, specification) +function input.locatedatabase(specification) + return input.methodhandler('locators', specification) end -function input.locators.tex(instance,specification) +function input.locators.tex(specification) if specification and specification ~= '' and lfs.isdir(specification) then - input.logger('! tex locator', specification..' found') - input.aux.append_hash(instance,'file',specification,filename) - else - input.logger('? tex locator', specification..' not found') + if input.trace > 0 then + input.logger('! tex locator found: %s',specification) + end + input.aux.append_hash('file',specification,filename) + elseif input.trace > 0 then + input.logger('? tex locator not found: %s',specification) end end -- hashers -function input.hashdatabase(instance,tag,name) - return input.methodhandler('hashers',instance,tag,name) +function input.hashdatabase(tag,name) + return input.methodhandler('hashers',tag,name) end -function input.loadfiles(instance) +function input.loadfiles() + local instance = input.instance instance.loaderror = false instance.files = { } if not instance.renewcache then for _, hash in ipairs(instance.hashes) do - input.hashdatabase(instance,hash.tag,hash.name) + input.hashdatabase(hash.tag,hash.name) if instance.loaderror then break end end end end -function input.hashers.tex(instance,tag,name) - input.aux.load_files(instance,tag) +function input.hashers.tex(tag,name) + input.aux.load_files(tag) end -- generators: -function input.loadlists(instance) - for _, hash in ipairs(instance.hashes) do - input.generatedatabase(instance,hash.tag) +function input.loadlists() + for _, hash in ipairs(input.instance.hashes) do + input.generatedatabase(hash.tag) end end -function input.generatedatabase(instance,specification) - return input.methodhandler('generators', instance, specification) -end - -do - - local weird = lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) - - function input.generators.tex(instance,specification) - local tag = specification - if not instance.lsrmode and lfs and lfs.dir then - input.report("scanning path",specification) - instance.files[tag] = { } - local files = instance.files[tag] - local n, m, r = 0, 0, 0 - local spec = specification .. '/' - local attributes = lfs.attributes - local directory = lfs.dir - local small = instance.smallcache - local function action(path) - local mode, full - if path then - full = spec .. path .. '/' +function input.generatedatabase(specification) + return input.methodhandler('generators', specification) +end + +local weird = lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) + +function input.generators.tex(specification) + local instance = input.instance + local tag = specification + if not instance.lsrmode and lfs.dir then + input.report("scanning path %s",specification) + instance.files[tag] = { } + local files = instance.files[tag] + local n, m, r = 0, 0, 0 + local spec = specification .. '/' + local attributes = lfs.attributes + local directory = lfs.dir + local small = instance.smallcache + local function action(path) + local mode, full + if path then + full = spec .. path .. '/' + else + full = spec + end + for name in directory(full) do + if name:find("^%.") then + -- skip + -- elseif name:find("[%~%`%!%#%$%%%^%&%*%(%)%=%{%}%[%]%:%;\"\'%|%<%>%,%?\n\r\t]") then -- too much escaped + elseif weird:match(name) then + -- texio.write_nl("skipping " .. name) + -- skip else - full = spec - end - for name in directory(full) do - if name:find("^%.") then - -- skip - -- elseif name:find("[%~%`%!%#%$%%%^%&%*%(%)%=%{%}%[%]%:%;\"\'%|%<%>%,%?\n\r\t]") then -- too much escaped - elseif weird:match(name) then - -- texio.write_nl("skipping " .. name) - -- skip - else - mode = attributes(full..name,'mode') - if mode == "directory" then - m = m + 1 - if path then - action(path..'/'..name) - else - action(name) - end - elseif path and mode == 'file' then - n = n + 1 - local f = files[name] - if f then - if not small then - if type(f) == 'string' then - files[name] = { f, path } - else - f[#f+1] = path - end - end - else - files[name] = path - local lower = name:lower() - if name ~= lower then - files["remap:"..lower] = name - r = r + 1 - end - end + mode = attributes(full..name,'mode') + if mode == 'directory' then + m = m + 1 + if path then + action(path..'/'..name) + else + action(name) end - end - end - end - action() - input.report(format("%s files found on %s directories with %s uppercase remappings",n,m,r)) - else - local fullname = file.join(specification,input.lsrname) - local path = '.' - local f = io.open(fullname) - if f then - instance.files[tag] = { } - local files = instance.files[tag] - local small = instance.smallcache - input.report("loading lsr file",fullname) - -- for line in f:lines() do -- much slower then the next one - for line in (f:read("*a")):gmatch("(.-)\n") do - if line:find("^[%a%d]") then - local fl = files[line] - if fl then + elseif path and mode == 'file' then + n = n + 1 + local f = files[name] + if f then if not small then - if type(fl) == 'string' then - files[line] = { fl, path } -- table + if type(f) == 'string' then + files[name] = { f, path } else - fl[#fl+1] = path + f[#f+1] = path end end else - files[line] = path -- string - local lower = line:lower() - if line ~= lower then - files["remap:"..lower] = line + files[name] = path + local lower = name:lower() + if name ~= lower then + files["remap:"..lower] = name + r = r + 1 + end + end + end + end + end + end + action() + input.report("%s files found on %s directories with %s uppercase remappings",n,m,r) + else + local fullname = file.join(specification,input.lsrname) + local path = '.' + local f = io.open(fullname) + if f then + instance.files[tag] = { } + local files = instance.files[tag] + local small = instance.smallcache + input.report("loading lsr file %s",fullname) + -- for line in f:lines() do -- much slower then the next one + for line in (f:read("*a")):gmatch("(.-)\n") do + if line:find("^[%a%d]") then + local fl = files[line] + if fl then + if not small then + if type(fl) == 'string' then + files[line] = { fl, path } -- table + else + fl[#fl+1] = path end end else - path = line:match("%.%/(.-)%:$") or path -- match could be nil due to empty line + files[line] = path -- string + local lower = line:lower() + if line ~= lower then + files["remap:"..lower] = line + end end + else + path = line:match("%.%/(.-)%:$") or path -- match could be nil due to empty line end - f:close() end + f:close() end end - end -- savers, todo -function input.savefiles(instance) - input.aux.save_data(instance, 'files', function(k,v) - return instance.validfile(k,v) -- path, name +function input.savefiles() + input.aux.save_data('files', function(k,v) + return input.instance.validfile(k,v) -- path, name end) end @@ -754,8 +781,8 @@ end -- we join them and split them after the expansion has taken place. This -- is more convenient. -function input.splitconfig(instance) - for i,c in ipairs(instance) do +function input.splitconfig() + for i,c in ipairs(input.instance) do for k,v in pairs(c) do if type(v) == 'string' then local t = file.split_path(v) @@ -766,8 +793,9 @@ function input.splitconfig(instance) end end end -function input.joinconfig(instance) - for i,c in ipairs(instance.order) do + +function input.joinconfig() + for i,c in ipairs(input.instance.order) do for k,v in pairs(c) do if type(v) == 'table' then c[k] = file.join_path(v) @@ -790,8 +818,9 @@ function input.join_path(str) end end -function input.splitexpansions(instance) - for k,v in pairs(instance.expansions) do +function input.splitexpansions() + local ie = input.instance.expansions + for k,v in pairs(ie) do local t, h = { }, { } for _,vv in pairs(file.split_path(v)) do if vv ~= "" and not h[vv] then @@ -800,19 +829,19 @@ function input.splitexpansions(instance) end end if #t > 1 then - instance.expansions[k] = t + ie[k] = t else - instance.expansions[k] = t[1] + ie[k] = t[1] end end end -- end of split/join code -function input.saveoldconfig(instance) - input.splitconfig(instance) - input.aux.save_data(instance, 'configuration', nil) - input.joinconfig(instance) +function input.saveoldconfig() + input.splitconfig() + input.aux.save_data('configuration', nil) + input.joinconfig() end input.configbanner = [[ @@ -829,8 +858,6 @@ function input.serialize(files) -- luatools and mtxtools are called frequently. Okay, -- we pay a small price for properly tabbed tables. local t = { } - local concat = table.concat - local sorted = table.sortedkeys local function dump(k,v,m) if type(v) == 'string' then return m .. "['" .. k .. "']='" .. v .. "'," @@ -841,12 +868,12 @@ function input.serialize(files) end end t[#t+1] = "return {" - if instance.sortdata then - for _, k in pairs(sorted(files)) do + if input.instance.sortdata then + for _, k in pairs(sortedkeys(files)) do local fk = files[k] if type(fk) == 'table' then t[#t+1] = "\t['" .. k .. "']={" - for _, kk in pairs(sorted(fk)) do + for _, kk in pairs(sortedkeys(fk)) do t[#t+1] = dump(kk,fk[kk],"\t\t") end t[#t+1] = "\t}," @@ -873,11 +900,11 @@ end if not texmf then texmf = {} end -- no longer needed, at least not here -function input.aux.save_data(instance, dataname, check, makename) -- untested without cache overload - for cachename, files in pairs(instance[dataname]) do +function input.aux.save_data(dataname, check, makename) -- untested without cache overload + for cachename, files in pairs(input.instance[dataname]) do local name = (makename or file.join)(cachename,dataname) - local luaname, lucname = name .. input.luasuffix, name .. input.lucsuffix - input.report("preparing " .. dataname .. " for", luaname) + local luaname, lucname = name .. ".lua", name .. ".luc" + input.report("preparing %s for %s",dataname,cachename) for k, v in pairs(files) do if not check or check(v,k) then -- path, name if type(v) == "table" and #v == 1 then @@ -895,38 +922,38 @@ function input.aux.save_data(instance, d time = os.date("%H:%M:%S"), content = files, } - local f = io.open(luaname,'w') - if f then - input.report("saving " .. dataname .. " in", luaname) - f:write(input.serialize(data)) - f:close() - input.report("compiling " .. dataname .. " to", lucname) - if not utils.lua.compile(luaname,lucname) then - input.report("compiling failed for " .. dataname .. ", deleting file " .. lucname) + local ok = io.savedata(luaname,input.serialize(data)) + if ok then + input.report("%s saved in %s",dataname,luaname) + if utils.lua.compile(luaname,lucname,false,true) then -- no cleanup but strip + input.report("%s compiled to %s",dataname,lucname) + else + input.report("compiling failed for %s, deleting file %s",dataname,lucname) os.remove(lucname) end else - input.report("unable to save " .. dataname .. " in " .. name..input.luasuffix) + input.report("unable to save %s in %s (access error)",dataname,luaname) end end end -function input.aux.load_data(instance,pathname,dataname,filename,makename) -- untested without cache overload +function input.aux.load_data(pathname,dataname,filename,makename) -- untested without cache overload + local instance = input.instance filename = ((not filename or (filename == "")) and dataname) or filename filename = (makename and makename(dataname,filename)) or file.join(pathname,filename) - local blob = loadfile(filename .. input.lucsuffix) or loadfile(filename .. input.luasuffix) + local blob = loadfile(filename .. ".luc") or loadfile(filename .. ".lua") if blob then local data = blob() if data and data.content and data.type == dataname and data.version == input.cacheversion then - input.report("loading",dataname,"for",pathname,"from",filename) + input.report("loading %s for %s from %s",dataname,pathname,filename) instance[dataname][pathname] = data.content else - input.report("skipping",dataname,"for",pathname,"from",filename) + input.report("skipping %s for %s from %s",dataname,pathname,filename) instance[dataname][pathname] = { } instance.loaderror = true end else - input.report("skipping",dataname,"for",pathname,"from",filename) + input.report("skipping %s for %s from %s",dataname,pathname,filename) end end @@ -939,13 +966,14 @@ end -- TEXMFBOGUS = 'effe checken of dit werkt', -- } -function input.aux.load_texmfcnf(instance,dataname,pathname) +function input.aux.load_texmfcnf(dataname,pathname) + local instance = input.instance local filename = file.join(pathname,input.luaname) local blob = loadfile(filename) if blob then local data = blob() if data then - input.report("loading","configuration file",filename) + input.report("loading configuration file %s",filename) if true then -- flatten to variable.progname local t = { } @@ -965,172 +993,168 @@ function input.aux.load_texmfcnf(instanc instance[dataname][pathname] = data end else - input.report("skipping","configuration file",filename) + input.report("skipping configuration file %s",filename) instance[dataname][pathname] = { } instance.loaderror = true end else - input.report("skipping","configuration file",filename) + input.report("skipping configuration file %s",filename) end end -function input.aux.load_configuration(instance,dname,lname) - input.aux.load_data(instance,dname,'configuration',lname and file.basename(lname)) +function input.aux.load_configuration(dname,lname) + input.aux.load_data(dname,'configuration',lname and file.basename(lname)) end -function input.aux.load_files(instance,tag) - input.aux.load_data(instance,tag,'files') +function input.aux.load_files(tag) + input.aux.load_data(tag,'files') end -function input.resetconfig(instance) +function input.resetconfig() + input.identify_own() + local instance = input.instance instance.configuration, instance.setup, instance.order, instance.loaderror = { }, { }, { }, false end -function input.loadnewconfig(instance) +function input.loadnewconfig() + local instance = input.instance for _, cnf in ipairs(instance.luafiles) do local dname = file.dirname(cnf) - input.aux.load_texmfcnf(instance,'setup',dname) + input.aux.load_texmfcnf('setup',dname) instance.order[#instance.order+1] = instance.setup[dname] if instance.loaderror then break end end end -function input.loadoldconfig(instance) +function input.loadoldconfig() + local instance = input.instance if not instance.renewcache then for _, cnf in ipairs(instance.cnffiles) do local dname = file.dirname(cnf) - input.aux.load_configuration(instance,dname) + input.aux.load_configuration(dname) instance.order[#instance.order+1] = instance.configuration[dname] if instance.loaderror then break end end end - input.joinconfig(instance) + input.joinconfig() end -function input.expand_variables(instance) - instance.expansions = { } ---~ instance.environment['SELFAUTOPARENT'] = instance.environment['SELFAUTOPARENT'] or instance.rootpath - if instance.engine ~= "" then instance.environment['engine'] = instance.engine end - if instance.progname ~= "" then instance.environment['progname'] = instance.progname end - for k,v in pairs(instance.environment) do +function input.expand_variables() + local instance = input.instance + local expansions, environment, variables = { }, instance.environment, instance.variables + local env = input.env + instance.expansions = expansions + if instance.engine ~= "" then environment['engine'] = instance.engine end + if instance.progname ~= "" then environment['progname'] = instance.progname end + for k,v in pairs(environment) do local a, b = k:match("^(%a+)%_(.*)%s*$") if a and b then - instance.expansions[a..'.'..b] = v + expansions[a..'.'..b] = v else - instance.expansions[k] = v + expansions[k] = v end end - for k,v in pairs(instance.environment) do -- move environment to expansions - if not instance.expansions[k] then instance.expansions[k] = v end + for k,v in pairs(environment) do -- move environment to expansions + if not expansions[k] then expansions[k] = v end end - for k,v in pairs(instance.variables) do -- move variables to expansions - if not instance.expansions[k] then instance.expansions[k] = v end + for k,v in pairs(variables) do -- move variables to expansions + if not expansions[k] then expansions[k] = v end end while true do local busy = false - for k,v in pairs(instance.expansions) do + for k,v in pairs(expansions) do local s, n = v:gsub("%$([%a%d%_%-]+)", function(a) busy = true - return instance.expansions[a] or input.env(instance,a) + return expansions[a] or env(a) end) local s, m = s:gsub("%$%{([%a%d%_%-]+)%}", function(a) busy = true - return instance.expansions[a] or input.env(instance,a) + return expansions[a] or env(a) end) if n > 0 or m > 0 then - instance.expansions[k]= s + expansions[k]= s end end if not busy then break end end - local homedir = - instance.environment[(os.type == "windows" and 'USERPROFILE') or 'HOME'] or '~' - for k,v in pairs(instance.expansions) do - v = v:gsub("^~", homedir) - instance.expansions[k] = v:gsub("\\", '/') + for k,v in pairs(expansions) do + expansions[k] = v:gsub("\\", '/') end end -function input.aux.expand_vars(instance,lst) -- simple vars +function input.aux.expand_vars(lst) -- simple vars + local instance = input.instance + local variables, env = instance.variables, input.env for k,v in pairs(lst) do lst[k] = v:gsub("%$([%a%d%_%-]+)", function(a) - return instance.variables[a] or input.env(instance,a) + return variables[a] or env(a) end) end end -function input.aux.expanded_var(instance,var) -- simple vars +function input.aux.expanded_var(var) -- simple vars + local instance = input.instance return var:gsub("%$([%a%d%_%-]+)", function(a) - return instance.variables[a] or input.env(instance,a) + return instance.variables[a] or input.env(a) end) end -function input.aux.entry(instance,entries,name) +function input.aux.entry(entries,name) if name and (name ~= "") then + local instance = input.instance name = name:gsub('%$','') local result = entries[name..'.'..instance.progname] or entries[name] if result then return result else - result = input.env(instance,name) + result = input.env(name) if result then instance.variables[name] = result - input.expand_variables(instance) + input.expand_variables() return instance.expansions[name] or "" end end end return "" end -function input.variable(instance,name) - return input.aux.entry(instance,instance.variables,name) +function input.variable(name) + return input.aux.entry(input.instance.variables,name) end -function input.expansion(instance,name) - return input.aux.entry(instance,instance.expansions,name) +function input.expansion(name) + return input.aux.entry(input.instance.expansions,name) end -function input.aux.is_entry(instance,entries,name) +function input.aux.is_entry(entries,name) if name and name ~= "" then name = name:gsub('%$','') - return (entries[name..'.'..instance.progname] or entries[name]) ~= nil + return (entries[name..'.'..input.instance.progname] or entries[name]) ~= nil else return false end end -function input.is_variable(instance,name) - return input.aux.is_entry(instance,instance.variables,name) -end -function input.is_expansion(instance,name) - return input.aux.is_entry(instance,instance.expansions,name) +function input.is_variable(name) + return input.aux.is_entry(input.instance.variables,name) end -function input.simplified_list(str) - if type(str) == 'table' then - return str -- troubles ; ipv , in texmf - elseif str == '' then - return { } - else - local t = { } - for _,v in ipairs(string.splitchr(str:gsub("^\{(.+)\}$","%1"),",")) do - t[#t+1] = (v:gsub("^[%!]*(.+)[%/\\]*$","%1")) - end - return t - end +function input.is_expansion(name) + return input.aux.is_entry(input.instance.expansions,name) end -function input.unexpanded_path_list(instance,str) - local pth = input.variable(instance,str) +function input.unexpanded_path_list(str) + local pth = input.variable(str) local lst = input.split_path(pth) - return input.aux.expanded_path(instance,lst) + return input.aux.expanded_path(lst) end -function input.unexpanded_path(instance,str) - return file.join_path(input.unexpanded_path_list(instance,str)) + +function input.unexpanded_path(str) + return file.join_path(input.unexpanded_path_list(str)) end do local done = { } - function input.reset_extra_path(instance) + function input.reset_extra_path() + local instance = input.instance local ep = instance.extra_paths if not ep then ep, done = { }, { } @@ -1140,7 +1164,8 @@ do end end - function input.register_extra_path(instance,paths,subpaths) + function input.register_extra_path(paths,subpaths) + local instance = input.instance local ep = instance.extra_paths or { } local n = #ep if paths and paths ~= "" then @@ -1185,7 +1210,8 @@ do end -function input.expanded_path_list(instance,str) +function input.expanded_path_list(str) + local instance = input.instance local function made_list(list) local ep = instance.extra_paths if not ep or #ep == 0 then @@ -1226,39 +1252,41 @@ function input.expanded_path_list(instan -- engine+progname hash str = str:gsub("%$","") if not instance.lists[str] then -- cached - local lst = made_list(input.split_path(input.expansion(instance,str))) - instance.lists[str] = input.aux.expanded_path(instance,lst) + local lst = made_list(input.split_path(input.expansion(str))) + instance.lists[str] = input.aux.expanded_path(lst) end return instance.lists[str] else - local lst = input.split_path(input.expansion(instance,str)) - return made_list(input.aux.expanded_path(instance,lst)) + local lst = input.split_path(input.expansion(str)) + return made_list(input.aux.expanded_path(lst)) end end -function input.expand_path(instance,str) - return file.join_path(input.expanded_path_list(instance,str)) + +function input.clean_path_list(str) + local t = input.expanded_path_list(str) + if t then + for i=1,#t do + t[i] = file.collapse_path(input.clean_path(t[i])) + end + end + return t end ---~ function input.first_writable_path(instance,name) ---~ for _,v in pairs(input.expanded_path_list(instance,name)) do ---~ if file.is_writable(file.join(v,'luatex-cache.tmp')) then ---~ return v ---~ end ---~ end ---~ return "." ---~ end +function input.expand_path(str) + return file.join_path(input.expanded_path_list(str)) +end -function input.expanded_path_list_from_var(instance,str) -- brrr +function input.expanded_path_list_from_var(str) -- brrr local tmp = input.var_of_format_or_suffix(str:gsub("%$","")) if tmp ~= "" then - return input.expanded_path_list(instance,str) + return input.expanded_path_list(str) else - return input.expanded_path_list(instance,tmp) + return input.expanded_path_list(tmp) end end -function input.expand_path_from_var(instance,str) - return file.join_path(input.expanded_path_list_from_var(instance,str)) +function input.expand_path_from_var(str) + return file.join_path(input.expanded_path_list_from_var(str)) end function input.format_of_var(str) @@ -1288,9 +1316,9 @@ function input.var_of_format_or_suffix(s return '' end -function input.expand_braces(instance,str) -- output variable and brace expansion of STRING - local ori = input.variable(instance,str) - local pth = input.aux.expanded_path(instance,input.split_path(ori)) +function input.expand_braces(str) -- output variable and brace expansion of STRING + local ori = input.variable(str) + local pth = input.aux.expanded_path(input.split_path(ori)) return file.join_path(pth) end @@ -1305,6 +1333,7 @@ end -- {a,b,c/{p,q,r}/d/{x,y,z}//} -- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}} -- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}} +-- {$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c} -- this one is better and faster, but it took me a while to realize -- that this kind of replacement is cleaner than messy parsing and @@ -1313,19 +1342,19 @@ end -- work that well; the parsing is ok, but dealing with the resulting -- table is a pain because we need to work inside-out recursively --- get rid of piecewise here, just a gmatch is ok - function input.aux.splitpathexpr(str, t, validate) -- no need for optimization, only called a few times, we can use lpeg for the sub t = t or { } - local concat = table.concat + str = str:gsub(",}",",@}") + str = str:gsub("{,","{@,") + -- str = "@" .. str .. "@" while true do local done = false while true do local ok = false - str = str:gsub("([^{},]+){([^{}]-)}", function(a,b) + str = str:gsub("([^{},]+){([^{}]+)}", function(a,b) local t = { } - b:piecewise(",", function(s) t[#t+1] = a .. s end) + for s in b:gmatch("[^,]+") do t[#t+1] = a .. s end ok, done = true, true return "{" .. concat(t,",") .. "}" end) @@ -1333,9 +1362,9 @@ function input.aux.splitpathexpr(str, t, end while true do local ok = false - str = str:gsub("{([^{}]-)}([^{},]+)", function(a,b) + str = str:gsub("{([^{}]+)}([^{},]+)", function(a,b) local t = { } - a:piecewise(",", function(s) t[#t+1] = s .. b end) + for s in a:gmatch("[^,]+") do t[#t+1] = s .. b end ok, done = true, true return "{" .. concat(t,",") .. "}" end) @@ -1343,50 +1372,41 @@ function input.aux.splitpathexpr(str, t, end while true do local ok = false - str = str:gsub("([,{]){([^{}]+)}([,}])", function(a,b,c) + str = str:gsub("{([^{}]+)}{([^{}]+)}", function(a,b) + local t = { } + for sa in a:gmatch("[^,]+") do + for sb in b:gmatch("[^,]+") do + t[#t+1] = sa .. sb + end + end ok, done = true, true - return a .. b .. c + return "{" .. concat(t,",") .. "}" end) if not ok then break end end - if not done then break end - end - while true do - local ok = false - str = str:gsub("{([^{}]-)}{([^{}]-)}", function(a,b) - local t = { } - a:piecewise(",", function(sa) - b:piecewise(",", function(sb) - t[#t+1] = sa .. sb - end) - end) - ok = true - return "{" .. concat(t,",") .. "}" - end) - if not ok then break end - end - while true do - local ok = false - str = str:gsub("{([^{}]-)}", function(a) - ok = true - return a + str = str:gsub("({[^{}]*){([^{}]+)}([^{}]*})", function(a,b,c) + done = true + return a .. b.. c end) - if not ok then break end + if not done then break end end + str = str:gsub("[{}]", "") + str = str:gsub("@","") if validate then - str:piecewise(",", function(s) + for s in str:gmatch("[^,]+") do s = validate(s) if s then t[#t+1] = s end - end) + end else - str:piecewise(",", function(s) + for s in str:gmatch("[^,]+") do t[#t+1] = s - end) + end end return t end -function input.aux.expanded_path(instance,pathlist) -- maybe not a list, just a path +function input.aux.expanded_path(pathlist) -- maybe not a list, just a path + local instance = input.instance -- a previous version fed back into pathlist local newlist, ok = { }, false for _,v in ipairs(pathlist) do @@ -1418,17 +1438,16 @@ input.is_readable = { } function input.aux.is_readable(readable, name) if input.trace > 2 then if readable then - input.logger("+ readable", name) + input.logger("+ readable: %s",name) else - input.logger("- readable", name) + input.logger("- readable: %s", name) end end return readable end function input.is_readable.file(name) - -- return input.aux.is_readable(file.is_readable(name), name) - return input.aux.is_readable(input.aux.is_file(name), name) + return input.aux.is_readable(lfs.isfile(name), name) end input.is_readable.tex = input.is_readable.file @@ -1436,12 +1455,13 @@ input.is_readable.tex = input.is_readabl -- name -- name/name -function input.aux.collect_files(instance,names) +function input.aux.collect_files(names) + local instance = input.instance local filelist = { } for _, fname in pairs(names) do if fname then if input.trace > 2 then - input.logger("? blobpath asked",fname) + input.logger("? blobpath asked: %s",fname) end local bname = file.basename(fname) local dname = file.dirname(fname) @@ -1455,7 +1475,7 @@ function input.aux.collect_files(instanc local files = blobpath and instance.files[blobpath] if files then if input.trace > 2 then - input.logger('? blobpath do',blobpath .. " (" .. bname ..")") + input.logger('? blobpath do: %s (%s)',blobpath,bname) end local blobfile = files[bname] if not blobfile then @@ -1488,7 +1508,7 @@ function input.aux.collect_files(instanc end end elseif input.trace > 1 then - input.logger('! blobpath no',blobpath .. " (" .. bname ..")" ) + input.logger('! blobpath no: %s (%s)',blobpath,bname) end end end @@ -1543,15 +1563,17 @@ do end -function input.aux.register_in_trees(instance,name) +function input.aux.register_in_trees(name) if not name:find("^%.") then + local instance = input.instance instance.foundintrees[name] = (instance.foundintrees[name] or 0) + 1 -- maybe only one end end -- split the next one up, better for jit -function input.aux.find_file(instance,filename) -- todo : plugin (scanners, checkers etc) +function input.aux.find_file(filename) -- todo : plugin (scanners, checkers etc) + local instance = input.instance local result = { } local stamp = nil filename = input.normalize_name(filename) -- elsewhere @@ -1560,16 +1582,22 @@ function input.aux.find_file(instance,fi if instance.remember then stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format if instance.found[stamp] then - input.logger('! remembered', filename) + if input.trace > 0 then + input.logger('! remembered: %s',filename) + end return instance.found[stamp] end end if filename:find('%*') then - input.logger('! wildcard', filename) - result = input.find_wildcard_files(instance,filename) + if input.trace > 0 then + input.logger('! wildcard: %s', filename) + end + result = input.find_wildcard_files(filename) elseif input.aux.qualified_path(filename) then if input.is_readable.file(filename) then - input.logger('! qualified', filename) + if input.trace > 0 then + input.logger('! qualified: %s', filename) + end result = { filename } else local forcedname, ok = "", false @@ -1577,22 +1605,26 @@ function input.aux.find_file(instance,fi if instance.format == "" then forcedname = filename .. ".tex" if input.is_readable.file(forcedname) then - input.logger('! no suffix, forcing standard filetype tex') + if input.trace > 0 then + input.logger('! no suffix, forcing standard filetype: tex') + end result, ok = { forcedname }, true end else for _, s in pairs(input.suffixes_of_format(instance.format)) do forcedname = filename .. "." .. s if input.is_readable.file(forcedname) then - input.logger('! no suffix, forcing format filetype', s) + if input.trace > 0 then + input.logger('! no suffix, forcing format filetype: %s', s) + end result, ok = { forcedname }, true break end end end end - if not ok then - input.logger('? qualified', filename) + if not ok and input.trace > 0 then + input.logger('? qualified: %s', filename) end end else @@ -1610,10 +1642,14 @@ function input.aux.find_file(instance,fi local forcedname = filename .. '.tex' wantedfiles[#wantedfiles+1] = forcedname filetype = input.format_of_suffix(forcedname) - input.logger('! forcing filetype',filetype) + if input.trace > 0 then + input.logger('! forcing filetype: %s',filetype) + end else filetype = input.format_of_suffix(filename) - input.logger('! using suffix based filetype',filetype) + if input.trace > 0 then + input.logger('! using suffix based filetype: %s',filetype) + end end else if ext == "" then @@ -1622,16 +1658,18 @@ function input.aux.find_file(instance,fi end end filetype = instance.format - input.logger('! using given filetype',filetype) + if input.trace > 0 then + input.logger('! using given filetype: %s',filetype) + end end local typespec = input.variable_of_format(filetype) - local pathlist = input.expanded_path_list(instance,typespec) + local pathlist = input.expanded_path_list(typespec) if not pathlist or #pathlist == 0 then -- no pathlist, access check only / todo == wildcard if input.trace > 2 then - input.logger('? filename',filename) - input.logger('? filetype',filetype or '?') - input.logger('? wanted files',table.concat(wantedfiles," | ")) + input.logger('? filename: %s',filename) + input.logger('? filetype: %s',filetype or '?') + input.logger('? wanted files: %s',concat(wantedfiles," | ")) end for _, fname in pairs(wantedfiles) do if fname and input.is_readable.file(fname) then @@ -1641,7 +1679,7 @@ function input.aux.find_file(instance,fi end end -- this is actually 'other text files' or 'any' or 'whatever' - local filelist = input.aux.collect_files(instance,wantedfiles) + local filelist = input.aux.collect_files(wantedfiles) local fl = filelist and filelist[1] if fl then filename = fl[3] @@ -1650,12 +1688,12 @@ function input.aux.find_file(instance,fi end else -- list search - local filelist = input.aux.collect_files(instance,wantedfiles) + local filelist = input.aux.collect_files(wantedfiles) local doscan, recurse if input.trace > 2 then - input.logger('? filename',filename) - -- if pathlist then input.logger('? path list',table.concat(pathlist," | ")) end - -- if filelist then input.logger('? file list',table.concat(filelist," | ")) end + input.logger('? filename: %s',filename) + -- if pathlist then input.logger('? path list: %s',concat(pathlist," | ")) end + -- if filelist then input.logger('? file list: %s',concat(filelist," | ")) end end -- a bit messy ... esp the doscan setting here for _, path in pairs(pathlist) do @@ -1676,11 +1714,11 @@ function input.aux.find_file(instance,fi if f:find(expr) then -- input.debug('T',' '..f) if input.trace > 2 then - input.logger('= found in hash',f) + input.logger('= found in hash: %s',f) end --- todo, test for readable result[#result+1] = fl[3] - input.aux.register_in_trees(instance,f) -- for tracing used files + input.aux.register_in_trees(f) -- for tracing used files done = true if not instance.allresults then break end else @@ -1694,12 +1732,12 @@ function input.aux.find_file(instance,fi local pname = pathname:gsub("%.%*$",'') if not pname:find("%*") then local ppname = pname:gsub("/+$","") - if input.aux.can_be_dir(instance,ppname) then + if input.aux.can_be_dir(ppname) then for _, w in pairs(wantedfiles) do local fname = file.join(ppname,w) if input.is_readable.file(fname) then if input.trace > 2 then - input.logger('= found by scanning',fname) + input.logger('= found by scanning: %s',fname) end result[#result+1] = fname done = true @@ -1728,40 +1766,29 @@ function input.aux.find_file(instance,fi return result end -input.aux._find_file_ = input.aux.find_file +input.aux._find_file_ = input.aux.find_file -- frozen variant -function input.aux.find_file(instance,filename) -- maybe make a lowres cache too - local result = input.aux._find_file_(instance,filename) +function input.aux.find_file(filename) -- maybe make a lowres cache too + local result = input.aux._find_file_(filename) if #result == 0 then local lowered = filename:lower() if filename ~= lowered then - return input.aux._find_file_(instance,lowered) + return input.aux._find_file_(lowered) end end return result end -if lfs and lfs.isfile then - input.aux.is_file = lfs.isfile -- to be done: use this -else - input.aux.is_file = file.is_readable -end - -if lfs and lfs.isdir then - function input.aux.can_be_dir(instance,name) - if not instance.fakepaths[name] then - if lfs.isdir(name) then - instance.fakepaths[name] = 1 -- directory - else - instance.fakepaths[name] = 2 -- no directory - end +function input.aux.can_be_dir(name) + local instance = input.instance + if not instance.fakepaths[name] then + if lfs.isdir(name) then + instance.fakepaths[name] = 1 -- directory + else + instance.fakepaths[name] = 2 -- no directory end - return (instance.fakepaths[name] == 1) - end -else - function input.aux.can_be_dir() - return true end + return (instance.fakepaths[name] == 1) end if not input.concatinators then input.concatinators = { } end @@ -1769,7 +1796,8 @@ if not input.concatinators then input.c input.concatinators.tex = file.join input.concatinators.file = input.concatinators.tex -function input.find_files(instance,filename,filetype,mustexist) +function input.find_files(filename,filetype,mustexist) + local instance = input.instance if type(mustexist) == boolean then -- all set elseif type(filetype) == 'boolean' then @@ -1778,16 +1806,17 @@ function input.find_files(instance,filen filetype, mustexist = nil, false end instance.format = filetype or '' - local t = input.aux.find_file(instance,filename,true) + local t = input.aux.find_file(filename,true) instance.format = '' return t end -function input.find_file(instance,filename,filetype,mustexist) - return (input.find_files(instance,filename,filetype,mustexist)[1] or "") +function input.find_file(filename,filetype,mustexist) + return (input.find_files(filename,filetype,mustexist)[1] or "") end -function input.find_given_files(instance,filename) +function input.find_given_files(filename) + local instance = input.instance local bname, result = file.basename(filename), { } for k, hash in ipairs(instance.hashes) do local files = instance.files[hash.tag] @@ -1815,11 +1844,12 @@ function input.find_given_files(instance return result end -function input.find_given_file(instance,filename) - return (input.find_given_files(instance,filename)[1] or "") +function input.find_given_file(filename) + return (input.find_given_files(filename)[1] or "") end -function input.find_wildcard_files(instance,filename) -- todo: remap: +function input.find_wildcard_files(filename) -- todo: remap: + local instance = input.instance local result = { } local bname, dname = file.basename(filename), file.dirname(filename) local path = dname:gsub("^*/","") @@ -1872,16 +1902,19 @@ function input.find_wildcard_files(insta if done and not allresults then break end end end + -- we can consider also searching the paths not in the database, but then + -- we end up with a messy search (all // in all path specs) return result end -function input.find_wildcard_file(instance,filename) - return (input.find_wildcard_files(instance,filename)[1] or "") +function input.find_wildcard_file(filename) + return (input.find_wildcard_files(filename)[1] or "") end -- main user functions -function input.save_used_files_in_trees(instance, filename,jobname) +function input.save_used_files_in_trees(filename,jobname) + local instance = input.instance if not filename then filename = 'luatex.jlg' end local f = io.open(filename,'w') if f then @@ -1891,7 +1924,7 @@ function input.save_used_files_in_trees( f:write("\t<rl:name>" .. jobname .. "</rl:name>\n") end f:write("\t<rl:files>\n") - for _,v in pairs(table.sortedkeys(instance.foundintrees)) do + for _,v in pairs(sorted(instance.foundintrees)) do -- ipairs f:write("\t\t<rl:file n='" .. instance.foundintrees[v] .. "'>" .. v .. "</rl:file>\n") end f:write("\t</rl:files>\n") @@ -1900,24 +1933,24 @@ function input.save_used_files_in_trees( end end -function input.automount(instance) +function input.automount() -- implemented later end -function input.load(instance) - input.starttiming(instance) - input.resetconfig(instance) - input.identify_cnf(instance) - input.load_lua(instance) - input.expand_variables(instance) - input.load_cnf(instance) - input.expand_variables(instance) - input.load_hash(instance) - input.automount(instance) - input.stoptiming(instance) +function input.load() + input.starttiming(input.instance) + input.resetconfig() + input.identify_cnf() + input.load_lua() + input.expand_variables() + input.load_cnf() + input.expand_variables() + input.load_hash() + input.automount() + input.stoptiming(input.instance) end -function input.for_files(instance, command, files, filetype, mustexist) +function input.for_files(command, files, filetype, mustexist) if files and #files > 0 then local function report(str) if input.verbose then @@ -1930,7 +1963,7 @@ function input.for_files(instance, comma report('') end for _, file in pairs(files) do - local result = command(instance,file,filetype,mustexist) + local result = command(file,filetype,mustexist) if type(result) == 'string' then report(result) else @@ -1944,14 +1977,11 @@ end -- strtab -function input.var_value(instance,str) -- output the value of variable $STRING. - return input.variable(instance,str) -end -function input.expand_var(instance,str) -- output variable expansion of STRING. - return input.expansion(instance,str) -end -function input.show_path(instance,str) -- output search path for file type NAME - return file.join_path(input.expanded_path_list(instance,input.format_of_var(str))) +input.var_value = input.variable -- output the value of variable $STRING. +input.expand_var = input.expansion -- output variable expansion of STRING. + +function input.show_path(str) -- output search path for file type NAME + return file.join_path(input.expanded_path_list(input.format_of_var(str))) end -- input.find_file(filename) @@ -2000,56 +2030,58 @@ function table.sequenced(t,sep) -- temp for k, v in pairs(t) do s[#s+1] = k .. "=" .. v end - return table.concat(s, sep or " | ") + return concat(s, sep or " | ") end -function input.methodhandler(what, instance, filename, filetype) -- ... +function input.methodhandler(what, filename, filetype) -- ... local specification = (type(filename) == "string" and input.splitmethod(filename)) or filename -- no or { }, let it bomb local scheme = specification.scheme if input[what][scheme] then - input.logger('= handler',specification.original .." -> " .. what .. " -> " .. table.sequenced(specification)) - return input[what][scheme](instance,filename,filetype) -- todo: specification + if input.trace > 0 then + input.logger('= handler: %s -> %s -> %s',specification.original,what,table.sequenced(specification)) + end + return input[what][scheme](filename,filetype) -- todo: specification else - return input[what].tex(instance,filename,filetype) -- todo: specification + return input[what].tex(filename,filetype) -- todo: specification end end -- also inside next test? -function input.findtexfile(instance, filename, filetype) - return input.methodhandler('finders',instance, input.normalize_name(filename), filetype) +function input.findtexfile(filename, filetype) + return input.methodhandler('finders',input.normalize_name(filename), filetype) end -function input.opentexfile(instance,filename) - return input.methodhandler('openers',instance, input.normalize_name(filename)) +function input.opentexfile(filename) + return input.methodhandler('openers',input.normalize_name(filename)) end -function input.findbinfile(instance, filename, filetype) - return input.methodhandler('finders',instance, input.normalize_name(filename), filetype) +function input.findbinfile(filename, filetype) + return input.methodhandler('finders',input.normalize_name(filename), filetype) end -function input.openbinfile(instance,filename) - return input.methodhandler('loaders',instance, input.normalize_name(filename)) +function input.openbinfile(filename) + return input.methodhandler('loaders',input.normalize_name(filename)) end -function input.loadbinfile(instance, filename, filetype) - local fname = input.findbinfile(instance, input.normalize_name(filename), filetype) +function input.loadbinfile(filename, filetype) + local fname = input.findbinfile(input.normalize_name(filename), filetype) if fname and fname ~= "" then - return input.openbinfile(instance,fname) + return input.openbinfile(fname) else return unpack(input.loaders.notfound) end end -function input.texdatablob(instance, filename, filetype) - local ok, data, size = input.loadbinfile(instance, filename, filetype) +function input.texdatablob(filename, filetype) + local ok, data, size = input.loadbinfile(filename, filetype) return data or "" end input.loadtexfile = input.texdatablob -function input.openfile(filename) -- brrr texmf.instance here / todo ! ! ! ! ! - local fullname = input.findtexfile(texmf.instance, filename) +function input.openfile(filename) + local fullname = input.findtexfile(filename) if fullname and (fullname ~= "") then - return input.opentexfile(texmf.instance, fullname) + return input.opentexfile(fullname) else return nil end @@ -2095,16 +2127,18 @@ end -- beware: i need to check where we still need a / on windows: function input.clean_path(str) ---~ return (((str:gsub("\\","/")):gsub("^!+","")):gsub("//+","//")) if str then - return ((str:gsub("\\","/")):gsub("^!+","")) + str = str:gsub("\\","/") + str = str:gsub("^!+","") + str = str:gsub("^~",input.homedir) + return str else return nil end end function input.do_with_path(name,func) - for _, v in pairs(input.expanded_path_list(instance,name)) do + for _, v in pairs(input.expanded_path_list(name)) do func("^"..input.clean_path(v)) end end @@ -2113,7 +2147,8 @@ function input.do_with_var(name,func) func(input.aux.expanded_var(name)) end -function input.with_files(instance,pattern,handle) +function input.with_files(pattern,handle) + local instance = input.instance for _, hash in ipairs(instance.hashes) do local blobpath = hash.tag local blobtype = hash.type @@ -2140,37 +2175,22 @@ function input.with_files(instance,patte end end ---~ function input.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix ---~ newname = file.addsuffix(newname,"lua") ---~ local newscript = input.clean_path(input.find_file(instance, newname)) ---~ local oldscript = input.clean_path(oldname) ---~ input.report("old script", oldscript) ---~ input.report("new script", newscript) ---~ if oldscript ~= newscript and (oldscript:find(file.removesuffix(newname).."$") or oldscript:find(newname.."$")) then ---~ local newdata = io.loaddata(newscript) ---~ if newdata then ---~ input.report("old script content replaced by new content") ---~ io.savedata(oldscript,newdata) ---~ end ---~ end ---~ end - -function input.update_script(instance,oldname,newname) -- oldname -> own.name, not per se a suffix +function input.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix local scriptpath = "scripts/context/lua" newname = file.addsuffix(newname,"lua") local oldscript = input.clean_path(oldname) - input.report("to be replaced old script", oldscript) - local newscripts = input.find_files(instance, newname) or { } + input.report("to be replaced old script %s", oldscript) + local newscripts = input.find_files(newname) or { } if #newscripts == 0 then input.report("unable to locate new script") else for _, newscript in ipairs(newscripts) do newscript = input.clean_path(newscript) - input.report("checking new script", newscript) + input.report("checking new script %s", newscript) if oldscript == newscript then input.report("old and new script are the same") elseif not newscript:find(scriptpath) then - input.report("new script should come from",scriptpath) + input.report("new script should come from %s",scriptpath) elseif not (oldscript:find(file.removesuffix(newname).."$") or oldscript:find(newname.."$")) then input.report("invalid new script name") else @@ -2198,10 +2218,10 @@ do local resolvers = { } - resolvers.environment = function(instance,str) + resolvers.environment = function(str) return input.clean_path(os.getenv(str) or os.getenv(str:upper()) or os.getenv(str:lower()) or "") end - resolvers.relative = function(instance,str,n) + resolvers.relative = function(str,n) if io.exists(str) then -- nothing elseif io.exists("./" .. str) then @@ -2219,16 +2239,16 @@ do end return input.clean_path(str) end - resolvers.locate = function(instance,str) - local fullname = input.find_given_file(instance,str) or "" + resolvers.locate = function(str) + local fullname = input.find_given_file(str) or "" return input.clean_path((fullname ~= "" and fullname) or str) end - resolvers.filename = function(instance,str) - local fullname = input.find_given_file(instance,str) or "" + resolvers.filename = function(str) + local fullname = input.find_given_file(str) or "" return input.clean_path(file.basename((fullname ~= "" and fullname) or str)) end - resolvers.pathname = function(instance,str) - local fullname = input.find_given_file(instance,str) or "" + resolvers.pathname = function(str) + local fullname = input.find_given_file(str) or "" return input.clean_path(file.dirname((fullname ~= "" and fullname) or str)) end @@ -2240,15 +2260,15 @@ do resolvers.file = resolvers.filename resolvers.path = resolvers.pathname - local function resolve(instance,str) + local function resolve(str) if type(str) == "table" then for k, v in pairs(str) do - str[k] = resolve(instance,v) or v + str[k] = resolve(v) or v end elseif str and str ~= "" then - str = str:gsub("([a-z]+):([^ ]+)", function(method,target) + str = str:gsub("([a-z]+):([^ \"\']*)", function(method,target) if resolvers[method] then - return resolvers[method](instance,target) + return resolvers[method](target) else return method .. ":" .. target end @@ -2257,6 +2277,24 @@ do return str end + if os.uname then + for k, v in pairs(os.uname()) do + if not resolvers[k] then + resolvers[k] = function() return v end + end + end + end + input.resolve = resolve end + +function input.boolean_variable(str,default) + local b = input.expansion(str) + if b == "" then + return default + else + b = toboolean(b) + return (b == nil and default) or b + end +end
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor
