wafadmin.Tools.preproc

1 #!/usr/bin/env python 2 # encoding: utf-8 3 # Thomas Nagy, 2006-2008 (ita) 4 5 #C/C++ preprocessor for finding dependencies 6 #TODO: more varargs, pragma once 7 8 import re, sys, os, string, types 9 if __name__ == '__main__': 10 sys.path = ['.', '..'] + sys.path 11 import Logs, Build, Utils 12 from Logs import debug, error 13 import traceback 14

15 -class PreprocError(Utils.WafError):

16 pass

17 18 go_absolute = 0 19 "set to 1 to track headers on files in /usr/include - else absolute paths are ignored" 20 21 standard_includes = ['/usr/include'] 22 if sys.platform == "win32": 23 standard_includes = [] 24 25 use_trigraphs = 0 26 'apply the trigraph rules first' 27 28 strict_quotes = 0 29 "Keep <> for system includes (do not search for those includes)" 30 31 g_optrans = { 32 'not':'!', 33 'and':'&&', 34 'bitand':'&', 35 'and_eq':'&=', 36 'or':'||', 37 'bitor':'|', 38 'or_eq':'|=', 39 'xor':'^', 40 'xor_eq':'^=', 41 'compl':'~', 42 } 43 "these ops are for c++, to reset, set an empty dict" 44 45 # ignore #warning and #error 46 re_lines = re.compile(\ 47 '^[ \t]*(#|%:)[ \t]*(ifdef|ifndef|if|else|elif|endif|include|import|define|undef|pragma)[ \t]*(.*)\r*$', 48 re.IGNORECASE | re.MULTILINE) 49 re_mac = re.compile("^[a-zA-Z_]\w*") 50 re_fun = re.compile('^[a-zA-Z_][a-zA-Z0-9_]*[(]') 51 re_pragma_once = re.compile('^\s*once\s*', re.IGNORECASE) 52 re_nl = re.compile('\\\\\r*\n', re.MULTILINE) 53 re_cpp = re.compile(\ 54 r"""(/\*[^*]*\*+([^/*][^*]*\*+)*/)|//[^\n]*|("(\\.|[^"\\])*"|'(\\.|[^'\\])*'|.[^/"'\\]*)""", 55 re.MULTILINE) 56 trig_def = [('??'+a, b) for a, b in zip("=-/!'()<>", r'#~\|^[]{}')] 57 chr_esc = {'0':0, 'a':7, 'b':8, 't':9, 'n':10, 'f':11, 'v':12, 'r':13, '\\':92, "'":39} 58 59 NUM = 'i' 60 OP = 'O' 61 IDENT = 'T' 62 STR = 's' 63 CHAR = 'c' 64 65 tok_types = [NUM, STR, IDENT, OP] 66 exp_types = [ 67 r"""0[xX](?P<hex>[a-fA-F0-9]+)(?P<qual1>[uUlL]*)|L*?'(?P<char>(\\.|[^\\'])+)'|(?P<n1>\d+)[Ee](?P<exp0>[+-]*?\d+)(?P<float0>[fFlL]*)|(?P<n2>\d*\.\d+)([Ee](?P<exp1>[+-]*?\d+))?(?P<float1>[fFlL]*)|(?P<n4>\d+\.\d*)([Ee](?P<exp2>[+-]*?\d+))?(?P<float2>[fFlL]*)|(?P<oct>0*)(?P<n0>\d+)(?P<qual2>[uUlL]*)""", 68 r'L?"([^"\\]|\\.)*"', 69 r'[a-zA-Z_]\w*', 70 r'%:%:|<<=|>>=|\.\.\.|<<|<%|<:|<=|>>|>=|\+\+|\+=|--|->|-=|\*=|/=|%:|%=|%>|==|&&|&=|\|\||\|=|\^=|:>|!=|##|[\{\}\[\]<>\?\|\^\*\+&=:!#;,%/\-\?\~\.]', 71 ] 72 re_clexer = re.compile('|'.join(["(?P<%s>%s)" % (name, part) for name, part in zip(tok_types, exp_types)]), re.M) 73 74 accepted = 'a' 75 ignored = 'i' 76 undefined = 'u' 77 skipped = 's' 78

79 -def repl(m):

80 s = m.group(1) 81 if s is not None: return ' ' 82 s = m.group(3) 83 if s is None: return '' 84 return s

85

86 -def filter_comments(filename):

87 # return a list of tuples : keyword, line 88 f = open(filename, "r") 89 code = f.read() 90 f.close() 91 if use_trigraphs: 92 for (a, b) in trig_def: code = code.split(a).join(b) 93 code = re_nl.sub('', code) 94 code = re_cpp.sub(repl, code) 95 return [(m.group(2), m.group(3)) for m in re.finditer(re_lines, code)]

96 97 prec = {} 98 # op -> number, needed for such expressions: #if 1 && 2 != 0 99 ops = ['* / %', '+ -', '<< >>', '< <= >= >', '== !=', '& | ^', '&& ||', ','] 100 for x in range(len(ops)): 101 syms = ops[x] 102 for u in syms.split(): 103 prec[u] = x 104

105 -def reduce_nums(val_1, val_2, val_op):

106 #print val_1, val_2, val_op 107 # pass two values, return a value 108 109 # now perform the operation, make certain a and b are numeric 110 try: a = 0 + val_1 111 except TypeError: a = int(val_1) 112 try: b = 0 + val_2 113 except TypeError: b = int(val_2) 114 115 d = val_op 116 if d == '%': c = a%b 117 elif d=='+': c = a+b 118 elif d=='-': c = a-b 119 elif d=='*': c = a*b 120 elif d=='/': c = a/b 121 elif d=='^': c = a^b 122 elif d=='|': c = a|b 123 elif d=='||': c = int(a or b) 124 elif d=='&': c = a&b 125 elif d=='&&': c = int(a and b) 126 elif d=='==': c = int(a == b) 127 elif d=='!=': c = int(a != b) 128 elif d=='<=': c = int(a <= b) 129 elif d=='<': c = int(a < b) 130 elif d=='>': c = int(a > b) 131 elif d=='>=': c = int(a >= b) 132 elif d=='^': c = int(a^b) 133 elif d=='<<': c = a<<b 134 elif d=='>>': c = a>>b 135 else: c = 0 136 return c

137

138 -def get_expr(lst, defs, ban):

139 140 if not lst: return ([], [], []) 141 142 (p, v) = lst[0] 143 if p == NUM: 144 return (p, v, lst[1:]) 145 146 elif p == STR: 147 try: 148 (p2, v2) = lst[1] 149 if p2 == STR: return (p, v+v2, lst[2:]) 150 except IndexError: pass 151 152 return (p, v, lst[1:]) 153 154 elif p == OP: 155 if v in ['+', '-', '!', '~', '#']: 156 (p2, v2, lst2) = get_expr(lst[1:], defs, ban) 157 158 if v == '#': 159 if p2 != IDENT: raise PreprocError, "ident expected %s" % str(lst) 160 return get_expr([(STR, v2)]+lst2, defs, ban) 161 162 if p2 != NUM: raise PreprocError, "num expected %s" % str(lst) 163 164 if v == '+': return (p2, v2, lst2) 165 elif v == '-': return (p2, - int(v2), lst2) 166 elif v == '!': return (p2, int(not int(v2)), lst2) 167 elif v == '~': return (p2, ~ int(v2), lst2) 168 169 return (p2, v2, lst2) 170 171 elif v == '(': 172 count_par = 0 173 i = 0 174 for _, v in lst: 175 if v == ')': 176 count_par -= 1 177 if count_par == 0: break 178 elif v == '(': count_par += 1 179 i += 1 180 else: 181 raise PreprocError, "rparen expected %s" % str(lst) 182 183 ret = process_tokens(lst[1:i], defs, ban) 184 if len(ret) == 1: 185 (p, v) = ret[0] 186 return (p, v, lst[i+1:]) 187 else: 188 #return (None, lst1, lst[i+1:]) 189 raise PreprocError, "cannot reduce %s" % str(lst) 190 191 elif p == IDENT: 192 if len(lst)>1: 193 (p2, v2) = lst[1] 194 if v2 == "##": 195 # token pasting, reevaluate the identifier obtained 196 (p3, v3) = lst[2] 197 if p3 != IDENT and p3 != NUM and p3 != OP: 198 raise PreprocError, "%s: ident expected after '##'" % str(lst) 199 return get_expr([(p, v+v3)]+lst[3:], defs, ban) 200 201 if v.lower() == 'defined': 202 (p2, v2) = lst[1] 203 off = 2 204 if v2 == '(': 205 (p3, v3) = lst[2] 206 if p3 != IDENT: raise PreprocError, 'expected an identifier after a "defined("' 207 (p2, v2) = lst[3] 208 if v2 != ')': raise PreprocError, 'expected a ")" after a "defined(x"' 209 off = 4 210 elif p2 != IDENT: 211 raise PreprocError, 'expected a "(" or an identifier after a defined' 212 213 x = 0 214 if v2 in defs: x = 1 215 #return get_expr([(NUM, x)] + lst[off:], defs, ban) 216 return (NUM, x, lst[off:]) 217 218 elif not v in defs or v in ban: 219 if "waf_include" in ban: return (p, v, lst[1:]) 220 else: return (NUM, 0, lst[1:]) 221 222 # tokenize on demand 223 if type(defs[v]) is types.StringType: 224 v, k = extract_macro(defs[v]) 225 defs[v] = k 226 macro_def = defs[v] 227 228 if not macro_def[0]: 229 # simple macro, substitute, and reevaluate 230 lst = macro_def[1] + lst[1:] 231 return get_expr(lst, defs, ban) 232 else: 233 # collect the arguments for the funcall 234 params = [] 235 i = 1 236 p2, v2 = lst[i] 237 if p2 != OP or v2 != '(': raise PreprocError, "invalid function call '%s'" % v 238 239 one_param = [] 240 count_paren = 0 241 try: 242 while 1: 243 i += 1 244 p2, v2 = lst[i] 245 246 if p2 == OP and count_paren == 0: 247 if v2 == '(': 248 one_param.append((p2, v2)) 249 count_paren += 1 250 elif v2 == ')': 251 if one_param: params.append(one_param) 252 lst = lst[i+1:] 253 break 254 elif v2 == ',': 255 if not one_param: raise PreprocError, "empty param in funcall %s" % p 256 params.append(one_param) 257 one_param = [] 258 else: 259 one_param.append((p2, v2)) 260 else: 261 one_param.append((p2, v2)) 262 if v2 == '(': count_paren += 1 263 elif v2 == ')': count_paren -= 1 264 265 except IndexError, e: 266 #raise PreprocError, 'invalid function call %s: missing ")"' % p 267 raise 268 269 # substitute the arguments within the define expression 270 accu = [] 271 table = macro_def[0] 272 for p2, v2 in macro_def[1]: 273 if p2 == IDENT and v2 in table: accu += params[table[v2]] 274 else: 275 if v2 == '__VA_ARGS__': 276 # first collect the tokens 277 va_toks = [] 278 st = len(macro_def[0]) 279 pt = len(params) 280 for x in params[pt-st+1:]: 281 va_toks.extend(x) 282 va_toks.append((OP, ',')) 283 if va_toks: va_toks.pop() # extra comma 284 if len(accu)>1: 285 (p3, v3) = accu[-1] 286 (p4, v4) = accu[-2] 287 if v3 == '##': 288 # remove the token paste 289 accu.pop() 290 if v4 == ',' and pt < st: 291 # remove the comma 292 accu.pop() 293 accu += va_toks 294 else: 295 accu.append((p2, v2)) 296 297 return get_expr(accu + lst, defs, ban+[v])

298

299 -def process_tokens(lst, defs, ban):

300 accu = [] 301 while lst: 302 p, v, nlst = get_expr(lst, defs, ban) 303 if p == NUM: 304 if not nlst: return [(p, v)] # finished 305 306 op1, ov1 = nlst[0] 307 if op1 != OP: 308 raise PreprocError, "op expected %s" % str(lst) 309 310 if ov1 == '?': 311 i = 0 312 count_par = 0 313 for _, k in nlst: 314 if k == ')': count_par -= 1 315 elif k == '(': count_par += 1 316 elif k == ':' and count_par == 0: break 317 i += 1 318 else: raise PreprocError, "ending ':' expected %s" % str(lst) 319 320 if reduce_nums(v, 0, '+'): lst = nlst[1:i] 321 else: lst = nlst[i+1:] 322 continue 323 324 elif ov1 == ',': 325 lst = nlst[1:] 326 continue 327 328 p2, v2, nlst = get_expr(nlst[1:], defs, ban) 329 if p2 != NUM: raise PreprocError, "num expected after op %s" % str(lst) 330 if nlst: 331 # op precedence 332 op3, ov3 = nlst[0] 333 if prec[ov3] < prec[ov1]: 334 #print "ov3", ov3, ov1 335 # as needed 336 p4, v4, nlst2 = get_expr(nlst[1:], defs, ban) 337 v5 = reduce_nums(v2, v4, ov3) 338 lst = [(p, v), (op1, ov1), (NUM, v5)] + nlst2 339 continue 340 341 # no op precedence or empty list, reduce the first tokens 342 lst = [(NUM, reduce_nums(v, v2, ov1))] + nlst 343 continue 344 345 elif p == STR: 346 if nlst: raise PreprocError, "sequence must terminate with a string %s" % str(nlst) 347 return [(p, v)] 348 349 return (None, None, [])

350

351 -def eval_macro(lst, adefs):

352 # look at the result, and try to return a 0/1 result 353 ret = process_tokens(lst, adefs, []) 354 if not ret: raise PreprocError, "missing tokens to evaluate %s" % str(lst) 355 p, v = ret[0] 356 return int(v) != 0

357

358 -class c_parser(object):

359 - def __init__(self, nodepaths=None, defines=None):

360 #self.lines = txt.split('\n') 361 self.lines = [] 362 363 if defines is None: 364 self.defs = {} 365 else: 366 self.defs = dict(defines) # make a copy 367

Source Code for Module wafadmin.Tools.preproc