File: Synopsis/Parsers/Python/ASGTranslator.py 1
2
3
4
5
6
7
8from Synopsis import ASG
9from Synopsis.QualifiedName import QualifiedPythonName as QName
10from Synopsis.SourceFile import *
11from Synopsis.DocString import DocString
12import sys, os.path
13import compiler, tokenize, token
14from compiler.consts import OP_ASSIGN
15from compiler.visitor import ASTVisitor
16
17class TokenParser:
18
19 def __init__(self, text):
20 self.text = text + '\n\n'
21 self.lines = self.text.splitlines(1)
22 self.generator = tokenize.generate_tokens(iter(self.lines).next)
23 self.next()
24
25 def __iter__(self):
26 return self
27
28 def next(self):
29 self.token = self.generator.next()
30 self.type, self.string, self.start, self.end, self.line = self.token
31 return self.token
32
33 def goto_line(self, lineno):
34 while self.start[0] < lineno:
35 self.next()
36 return token
37
38 def rhs(self, lineno):
39 """
40 Return a whitespace-normalized expression string from the right-hand
41 side of an assignment at line `lineno`.
42 """
43 self.goto_line(lineno)
44 while self.string != '=':
45 self.next()
46 self.stack = None
47 while self.type != token.NEWLINE and self.string != ';':
48 if self.string == '=' and not self.stack:
49 self.tokens = []
50 self.stack = []
51 self._type = None
52 self._string = None
53 self._backquote = 0
54 else:
55 self.note_token()
56 self.next()
57 self.next()
58 text = ''.join(self.tokens)
59 return text.strip()
60
61 closers = {')': '(', ']': '[', '}': '{'}
62 openers = {'(': 1, '[': 1, '{': 1}
63 del_ws_prefix = {'.': 1, '=': 1, ')': 1, ']': 1, '}': 1, ':': 1, ',': 1}
64 no_ws_suffix = {'.': 1, '=': 1, '(': 1, '[': 1, '{': 1}
65
66 def note_token(self):
67 if self.type == tokenize.NL:
68 return
69 del_ws = self.del_ws_prefix.has_key(self.string)
70 append_ws = not self.no_ws_suffix.has_key(self.string)
71 if self.openers.has_key(self.string):
72 self.stack.append(self.string)
73 if (self._type == token.NAME
74 or self.closers.has_key(self._string)):
75 del_ws = 1
76 elif self.closers.has_key(self.string):
77 assert self.stack[-1] == self.closers[self.string]
78 self.stack.pop()
79 elif self.string == '`':
80 if self._backquote:
81 del_ws = 1
82 assert self.stack[-1] == '`'
83 self.stack.pop()
84 else:
85 append_ws = 0
86 self.stack.append('`')
87 self._backquote = not self._backquote
88 if del_ws and self.tokens and self.tokens[-1] == ' ':
89 del self.tokens[-1]
90 self.tokens.append(self.string)
91 self._type = self.type
92 self._string = self.string
93 if append_ws:
94 self.tokens.append(' ')
95
96 def function_parameters(self, lineno):
97 """
98 Return a dictionary mapping parameters to defaults
99 (whitespace-normalized strings).
100 """
101 self.goto_line(lineno)
102 while self.string != 'def':
103 self.next()
104 while self.string != '(':
105 self.next()
106 name = None
107 default = False
108 parameter_tuple = False
109 self.tokens = []
110 parameters = {}
111 self.stack = [self.string]
112 self.next()
113 while 1:
114 if len(self.stack) == 1:
115 if parameter_tuple:
116 name = ''.join(self.tokens).strip()
117 self.tokens = []
118 parameter_tuple = False
119 if self.string in (')', ','):
120 if name:
121 if self.tokens:
122 default_text = ''.join(self.tokens).strip()
123 else:
124 default_text = ''
125 parameters[name] = default_text
126 self.tokens = []
127 name = None
128 default = False
129 if self.string == ')':
130 break
131 elif self.type == token.NAME:
132 if name and default:
133 self.note_token()
134 else:
135 assert name is None, (
136 'token=%r name=%r parameters=%r stack=%r'
137 % (self.token, name, parameters, self.stack))
138 name = self.string
139 elif self.string == '=':
140 assert name is not None, 'token=%r' % (self.token,)
141 assert default is False, 'token=%r' % (self.token,)
142 assert self.tokens == [], 'token=%r' % (self.token,)
143 default = True
144 self._type = None
145 self._string = None
146 self._backquote = 0
147 elif name:
148 self.note_token()
149 elif self.string == '(':
150 parameter_tuple = True
151 self._type = None
152 self._string = None
153 self._backquote = 0
154 self.note_token()
155 else:
156 assert (self.string in ('*', '**', '\n')
157 or self.type == tokenize.COMMENT), (
158 'token=%r' % (self.token,))
159 else:
160 self.note_token()
161 self.next()
162 return parameters
163
164class ASGTranslator(ASTVisitor):
165 """Translate the Python AST into a Synopsis ASG."""
166
167 def __init__(self, package, types, docformat):
168 """Create an ASGTranslator.
169
170 package: enclosing package the generated modules are to be part of."""
171
172 ASTVisitor.__init__(self)
173 self.scope = package and [package] or []
174 self.file = None
175 self.types = types
176 self.attributes = []
177 self.any_type = ASG.BuiltinTypeId('Python',QName('',))
178 self.docformat = docformat
179 self.documentable = None
180 self.name = QName()
181 self.imports = []
182 "Tuple with (module, names) pairs."
183
184 def process_file(self, file):
185
186 self.file = file
187 source = open(self.file.abs_name).read()
188 self.token_parser = TokenParser(source)
189 ast = compiler.parse(source)
190 compiler.walk(ast, self, walker=self)
191
192 def scope_name(self):
193 return len(self.scope) and self.scope[-1].name or ()
194
195 def default(self, node, *args):
196 self.documentable = None
197
198 def default_visit(self, node, *args):
199 ASTVisitor.default(self, node, *args)
200
201 def visitDiscard(self, node):
202 if self.documentable:
203 self.visit(node.expr)
204
205 def visitConst(self, node):
206 if self.documentable:
207 if type(node.value) in (str, unicode):
208 self.documentable.annotations['doc'] = DocString(node.value, self.docformat)
209 else:
210 self.documentable = None
211
212 def visitStmt(self, node):
213 self.default_visit(node)
214
215 def visitAssign(self, node):
216
217 save_attributes = self.attributes
218 self.attributes = []
219 self.in_ass_tuple = False
220 for child in node.nodes:
221 self.dispatch(child)
222 if self.attributes:
223 if type(self.scope[-1]) == ASG.Operation:
224
225 self.scope[-2].declarations.extend(self.attributes)
226 else:
227 self.scope[-1].declarations.extend(self.attributes)
228 if len(self.attributes) == 1:
229 self.documentable = self.attributes[0]
230 else:
231 self.documentable = None
232 self.attributes = save_attributes
233
234 def visitModule(self, node):
235
236 name = os.path.basename(os.path.splitext(self.file.name)[0])
237 if name == '__init__':
238 name = os.path.basename(os.path.dirname(self.file.name))
239 qname = QName(self.scope_name() + (name,))
240 module = ASG.Module(self.file, node.lineno, 'package', qname)
241 else:
242 qname = QName(self.scope_name() + (name,))
243 module = ASG.Module(self.file, node.lineno, 'module', qname)
244 self.types[qname] = ASG.DeclaredTypeId('Python', qname, module)
245
246 self.scope.append(module)
247 self.documentable = module
248 self.visit(node.node)
249 self.scope.pop()
250 self.file.declarations.append(module)
251
252 def visitImport(self, node):
253
254 self.imports.extend([(n[0], None) for n in node.names])
255 self.documentable = None
256
257 def visitFrom(self, node):
258
259 self.imports.extend([(node.modname, n[0]) for n in node.names])
260 self.documentable = None
261
262 def visitAssName(self, node):
263
264 if not self.in_ass_tuple:
265 meta_tags = ['__docformat__']
266 if len(self.scope) == 1 and node.name in meta_tags:
267 expression_text = eval(self.token_parser.rhs(node.lineno))
268 self.file.annotations[node.name] = expression_text
269 self.docformat = expression_text
270
271 qname = QName(self.scope_name() + (node.name,))
272 if type(self.scope[-1]) in (ASG.Function, ASG.Operation):
273 return
274 elif type(self.scope[-1]) == ASG.Class:
275 attribute = ASG.Variable(self.file, node.lineno, 'class attribute',
276 qname, self.any_type, False)
277 else:
278 attribute = ASG.Variable(self.file, node.lineno, 'attribute',
279 qname, self.any_type, False)
280 if node.name.startswith('__'):
281 attribute.accessibility = ASG.PRIVATE
282 elif node.name.startswith('_'):
283 attribute.accessibility = ASG.PROTECTED
284 self.attributes.append(attribute)
285 self.types[qname] = ASG.DeclaredTypeId('Python', attribute.name, attribute)
286
287 def visitAssTuple(self, node):
288
289 self.in_ass_tuple = True
290 for a in node.nodes:
291 self.visit(a)
292 self.in_ass_tuple = False
293
294 def visitAssAttr(self, node):
295 self.default_visit(node, node.attrname)
296 if type(self.scope[-1]) == ASG.Operation:
297
298
299
300 if self.name[0] == 'self':
301
302
303 qname = self.scope[-2].name + self.name[1:]
304 self.attributes.append(ASG.Variable(self.file, node.lineno,
305 'attribute', qname, self.any_type, False))
306
307 def visitGetattr(self, node, suffix):
308 self.default_visit(node, node.attrname + '.' + suffix)
309
310 def visitName(self, node, suffix=None):
311
312 if suffix:
313 self.name = QName((node.name,) + (suffix,))
314 else:
315 self.name = QName((node.name,))
316
317 def visitFunction(self, node):
318
319 if isinstance(self.scope[-1], ASG.Function):
320
321 return
322 qname = QName(self.scope_name() + (node.name,))
323 if type(self.scope[-1]) == ASG.Class:
324 function = ASG.Operation(self.file, node.lineno, 'method',
325 [], self.any_type, [], qname, node.name)
326 else:
327 function = ASG.Function(self.file, node.lineno, 'function',
328 [], self.any_type, [], qname, node.name)
329
330
331
332
333 special_attributes = ('__init__', '__str__', '__repr__', '__iter__', '__getitem__')
334
335 if node.name.startswith('__'):
336 if node.name not in special_attributes:
337 function.accessibility = ASG.PRIVATE
338 elif node.name.startswith('_'):
339 function.accessibility = ASG.PROTECTED
340
341 function.annotations['doc'] = DocString(node.doc or '', self.docformat)
342
343
344 self.types[qname] = ASG.DeclaredTypeId('Python', function.name, function)
345
346 self.scope.append(function)
347 self.documentable = function
348 function.parameters = self.parse_parameter_list(node)
349 if node.name == '__init__':
350
351 self.visit(node.code)
352 self.scope.pop()
353 self.scope[-1].declarations.append(function)
354
355 def parse_parameter_list(self, node):
356 parameters = []
357 special = []
358 argnames = list(node.argnames)
359 if node.kwargs:
360 special.append(ASG.Parameter('**', self.any_type, '', argnames[-1]))
361 argnames.pop()
362 if node.varargs:
363 special.append(ASG.Parameter('*', self.any_type, '', argnames[-1]))
364 argnames.pop()
365 defaults = list(node.defaults)
366 defaults = [None] * (len(argnames) - len(defaults)) + defaults
367 values = self.token_parser.function_parameters(node.lineno)
368 for argname, default in zip(node.argnames, defaults):
369 if type(argname) is tuple:
370 for a in argname:
371
372
373
374
375 parameters.append(ASG.Parameter('', self.any_type, '', a))
376 else:
377 parameters.append(ASG.Parameter('', self.any_type, '', argname,
378 values[argname]))
379 if parameters or special:
380 special.reverse()
381 parameters.extend(special)
382 return parameters
383
384 def visitClass(self, node):
385
386 if isinstance(self.scope[-1], ASG.Function):
387
388 return
389 bases = []
390 for base in node.bases:
391 self.visit(base)
392
393
394 if len(self.name) == 1 and self.scope:
395
396 base = QName(list(self.scope[-1].name) + list(self.name))
397 else:
398 base = self.name
399 if self.types.has_key(base):
400 base = self.types[base]
401 else:
402 base = ASG.UnknownTypeId('Python', base)
403 bases.append(base)
404 qname = QName(self.scope_name() + (node.name,))
405 class_ = ASG.Class(self.file, node.lineno, 'class', qname)
406 class_.parents = [ASG.Inheritance('', b, '') for b in bases]
407 class_.annotations['doc'] = DocString(node.doc or '', self.docformat)
408 self.types[qname] = ASG.DeclaredTypeId('Python', class_.name, class_)
409 self.scope.append(class_)
410 self.documentable = class_
411 self.visit(node.code)
412 self.scope.pop()
413 self.scope[-1].declarations.append(class_)
414
415 def visitGetattr(self, node, suffix=None):
416 if suffix:
417 name = node.attrname + '.' + suffix
418 else:
419 name = node.attrname
420 self.default_visit(node, name)
421
422
423
Generated on Thu Apr 16 16:27:16 2009 by
synopsis (version devel)