Coverage for adhoc-cicd-odoo-odoo / odoo / tools / cloc.py: 10%
207 statements
« prev ^ index » next coverage.py v7.13.4, created at 2026-03-09 18:15 +0000
1# -*- coding: utf-8 -*-
2# Part of Odoo. See LICENSE file for full copyright and licensing details.
3import ast
4import pathlib
5import os
6import re
7import shutil
9import odoo.modules
10from odoo import api
11from .config import config
13VERSION = 1
14DEFAULT_EXCLUDE = [
15 "__manifest__.py",
16 "__openerp__.py",
17 "tests/**/*",
18 "static/lib/**/*",
19 "static/tests/**/*",
20 "migrations/**/*",
21 "upgrades/**/*",
22]
24STANDARD_MODULES = ['web', 'web_enterprise', 'theme_common', 'base']
25MAX_FILE_SIZE = 25 * 2**20 # 25 MB
26MAX_LINE_SIZE = 100000
27VALID_EXTENSION = ['.py', '.js', '.xml', '.css', '.scss']
29class Cloc(object):
30 def __init__(self):
31 self.modules = {}
32 self.code = {}
33 self.total = {}
34 self.errors = {}
35 self.excluded = {}
36 self.max_width = 70
38 #------------------------------------------------------
39 # Parse
40 #------------------------------------------------------
41 def parse_xml(self, s):
42 s = s.strip() + "\n"
43 # Unbalanced xml comments inside a CDATA are not supported, and xml
44 # comments inside a CDATA will (wrongly) be considered as comment
45 total = s.count("\n")
46 s = re.sub("(<!--.*?-->)", "", s, flags=re.DOTALL)
47 s = re.sub(r"\s*\n\s*", r"\n", s).lstrip()
48 return s.count("\n"), total
50 def parse_py(self, s):
51 try:
52 s = s.strip() + "\n"
53 total = s.count("\n")
54 lines = set()
55 for i in ast.walk(ast.parse(s)):
56 # we only count 1 for a long string or a docstring
57 if hasattr(i, 'lineno'):
58 lines.add(i.lineno)
59 return len(lines), total
60 except Exception:
61 return (-1, "Syntax Error")
63 def parse_c_like(self, s, regex):
64 # Based on https://stackoverflow.com/questions/241327
65 s = s.strip() + "\n"
66 total = s.count("\n")
67 # To avoid to use too much memory we don't try to count file
68 # with very large line, usually minified file
69 if max(len(l) for l in s.split('\n')) > MAX_LINE_SIZE:
70 return -1, "Max line size exceeded"
72 def replacer(match):
73 s = match.group(0)
74 return " " if s.startswith('/') else s
76 comments_re = re.compile(regex, re.DOTALL | re.MULTILINE)
77 s = re.sub(comments_re, replacer, s)
78 s = re.sub(r"\s*\n\s*", r"\n", s).lstrip()
79 return s.count("\n"), total
81 def parse_js(self, s):
82 return self.parse_c_like(s, r'//.*?$|(?<!\\)/\*.*?\*/|\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"')
84 def parse_scss(self, s):
85 return self.parse_c_like(s, r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"')
87 def parse_css(self, s):
88 return self.parse_c_like(s, r'/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"')
90 def parse(self, s, ext):
91 if ext == '.py':
92 return self.parse_py(s)
93 elif ext == '.js':
94 return self.parse_js(s)
95 elif ext == '.xml':
96 return self.parse_xml(s)
97 elif ext == '.css':
98 return self.parse_css(s)
99 elif ext == '.scss':
100 return self.parse_scss(s)
102 #------------------------------------------------------
103 # Enumeration
104 #------------------------------------------------------
105 def book(self, module, item='', count=(0, 0), exclude=False):
106 if count[0] == -1:
107 self.errors.setdefault(module, {})
108 self.errors[module][item] = count[1]
109 elif exclude and item:
110 self.excluded.setdefault(module, {})
111 self.excluded[module][item] = count
112 else:
113 self.modules.setdefault(module, {})
114 if item:
115 self.modules[module][item] = count
116 self.code[module] = self.code.get(module, 0) + count[0]
117 self.total[module] = self.total.get(module, 0) + count[1]
118 self.max_width = max(self.max_width, len(module), len(item) + 4)
120 def count_path(self, path, exclude=None):
121 path = path.rstrip('/')
122 exclude_list = []
123 for i in odoo.modules.module.MANIFEST_NAMES:
124 manifest_path = os.path.join(path, i)
125 try:
126 with open(manifest_path, 'rb') as manifest:
127 exclude_list.extend(DEFAULT_EXCLUDE)
128 d = ast.literal_eval(manifest.read().decode('latin1'))
129 for j in ['cloc_exclude', 'demo', 'demo_xml']:
130 exclude_list.extend(d.get(j, []))
131 break
132 except Exception:
133 pass
134 if not exclude:
135 exclude = set()
136 for i in filter(None, exclude_list):
137 exclude.update(str(p) for p in pathlib.Path(path).glob(i))
139 module_name = os.path.basename(path)
140 self.book(module_name)
141 for root, _dirs, files in os.walk(path):
142 for file_name in files:
143 file_path = os.path.join(root, file_name)
145 if file_path in exclude:
146 continue
148 ext = os.path.splitext(file_path)[1].lower()
149 if ext not in VALID_EXTENSION:
150 continue
152 if os.path.getsize(file_path) > MAX_FILE_SIZE:
153 self.book(module_name, file_path, (-1, "Max file size exceeded"))
154 continue
156 with open(file_path, 'rb') as f:
157 # Decode using latin1 to avoid error that may raise by decoding with utf8
158 # The chars not correctly decoded in latin1 have no impact on how many lines will be counted
159 content = f.read().decode('latin1')
160 self.book(module_name, file_path, self.parse(content, ext))
162 def count_modules(self, env):
163 # Exclude standard addons paths
164 exclude_path = {
165 m.addons_path for name in STANDARD_MODULES
166 if (m := odoo.modules.Manifest.for_addon(name, display_warning=False))
167 }
169 domain = [('state', '=', 'installed')]
170 # if base_import_module is present
171 if env['ir.module.module']._fields.get('imported'):
172 domain.append(('imported', '=', False))
173 module_list = env['ir.module.module'].search(domain).mapped('name')
175 for module_name in module_list:
176 manifest = odoo.modules.Manifest.for_addon(module_name)
177 if manifest and manifest.addons_path not in exclude_path:
178 self.count_path(manifest.path)
180 def count_customization(self, env):
181 imported_module_sa = ""
182 if env['ir.module.module']._fields.get('imported'):
183 imported_module_sa = "OR (m.imported = TRUE AND m.state = 'installed')"
184 query = """
185 SELECT s.id, min(m.name), array_agg(d.module)
186 FROM ir_act_server AS s
187 LEFT JOIN ir_model_data AS d
188 ON (d.res_id = s.id AND d.model = 'ir.actions.server')
189 LEFT JOIN ir_module_module AS m
190 ON m.name = d.module
191 WHERE s.state = 'code' AND (m.name IS null {})
192 GROUP BY s.id
193 """.format(imported_module_sa)
194 env.cr.execute(query)
195 data = {r[0]: (r[1], r[2]) for r in env.cr.fetchall()}
196 for a in env['ir.actions.server'].browse(data.keys()):
197 self.book(
198 data[a.id][0] or "odoo/studio",
199 "ir.actions.server/%s: %s" % (a.id, a.name),
200 self.parse_py(a.code),
201 '__cloc_exclude__' in data[a.id][1]
202 )
204 imported_module_field = ("'odoo/studio'", "")
205 if env['ir.module.module']._fields.get('imported'):
206 imported_module_field = ("min(m.name)", "AND m.imported = TRUE AND m.state = 'installed'")
207 # We always want to count manual compute field unless they are generated by studio
208 # the module should be odoo/studio unless it comes from an imported module install
209 # because manual field get an external id from the original module of the model
210 query = r"""
211 SELECT f.id, f.name, {}, array_agg(d.module)
212 FROM ir_model_fields AS f
213 LEFT JOIN ir_model_data AS d ON (d.res_id = f.id AND d.model = 'ir.model.fields')
214 LEFT JOIN ir_module_module AS m ON m.name = d.module {}
215 WHERE f.compute IS NOT null AND f.state = 'manual'
216 GROUP BY f.id, f.name
217 """.format(*imported_module_field)
218 env.cr.execute(query)
219 # Do not count field generated by studio
220 all_data = env.cr.fetchall()
221 data = {r[0]: (r[2], r[3]) for r in all_data if not ("studio_customization" in r[3] and not r[1].startswith('x_studio'))}
222 for f in env['ir.model.fields'].browse(data.keys()):
223 self.book(
224 data[f.id][0] or "odoo/studio",
225 "ir.model.fields/%s: %s" % (f.id, f.name),
226 self.parse_py(f.compute),
227 '__cloc_exclude__' in data[f.id][1]
228 )
230 if not env['ir.module.module']._fields.get('imported'):
231 return
233 # Count qweb view only from imported module and not studio
234 query = """
235 SELECT view.id, min(mod.name), array_agg(data.module)
236 FROM ir_ui_view view
237 INNER JOIN ir_model_data data ON view.id = data.res_id AND data.model = 'ir.ui.view'
238 LEFT JOIN ir_module_module mod ON mod.name = data.module AND mod.imported = True
239 WHERE view.type = 'qweb' AND data.module != 'studio_customization'
240 GROUP BY view.id
241 HAVING count(mod.name) > 0
242 """
243 env.cr.execute(query)
244 custom_views = {r[0]: (r[1], r[2]) for r in env.cr.fetchall()}
245 for view in env['ir.ui.view'].browse(custom_views.keys()):
246 module_name = custom_views[view.id][0]
247 self.book(
248 module_name,
249 "/%s/views/%s.xml" % (module_name, view.name),
250 self.parse_xml(view.arch_base),
251 '__cloc_exclude__' in custom_views[view.id][1]
252 )
254 # Count js, xml, css/scss file from imported module
255 query = r"""
256 SELECT attach.id, min(mod.name), array_agg(data.module)
257 FROM ir_attachment attach
258 INNER JOIN ir_model_data data ON attach.id = data.res_id AND data.model = 'ir.attachment'
259 LEFT JOIN ir_module_module mod ON mod.name = data.module AND mod.imported = True
260 WHERE attach.name ~ '.*\.(js|xml|css|scss)$'
261 GROUP BY attach.id
262 HAVING count(mod.name) > 0
263 """
264 env.cr.execute(query)
265 uploaded_file = {r[0]: (r[1], r[2]) for r in env.cr.fetchall()}
266 for attach in env['ir.attachment'].browse(uploaded_file.keys()):
267 module_name = uploaded_file[attach.id][0]
268 ext = os.path.splitext(attach.url)[1].lower()
269 if ext not in VALID_EXTENSION:
270 continue
272 if len(attach.datas) > MAX_FILE_SIZE:
273 self.book(module_name, attach.url, (-1, "Max file size exceeded"))
274 continue
276 # Decode using latin1 to avoid error that may raise by decoding with utf8
277 # The chars not correctly decoded in latin1 have no impact on how many lines will be counted
278 content = attach.raw.decode('latin1')
279 self.book(
280 module_name,
281 attach.url,
282 self.parse(content, ext),
283 '__cloc_exclude__' in uploaded_file[attach.id][1],
284 )
286 def count_env(self, env):
287 self.count_modules(env)
288 self.count_customization(env)
290 def count_database(self, database):
291 registry = odoo.modules.registry.Registry(database)
292 with registry.cursor() as cr:
293 uid = api.SUPERUSER_ID
294 env = api.Environment(cr, uid, {})
295 self.count_env(env)
297 #------------------------------------------------------
298 # Report
299 #------------------------------------------------------
300 # pylint: disable=W0141
301 def report(self, verbose=False, width=None):
302 # Prepare format
303 if not width:
304 width = min(self.max_width, shutil.get_terminal_size()[0] - 24)
305 hr = "-" * (width + 24) + "\n"
306 fmt = '{k:%d}{lines:>8}{other:>8}{code:>8}\n' % (width,)
308 # Render
309 s = fmt.format(k="Odoo cloc", lines="Line", other="Other", code="Code")
310 s += hr
311 for m in sorted(self.modules):
312 s += fmt.format(k=m, lines=self.total[m], other=self.total[m]-self.code[m], code=self.code[m])
313 if verbose:
314 for i in sorted(self.modules[m], key=lambda i: self.modules[m][i][0], reverse=True):
315 code, total = self.modules[m][i]
316 s += fmt.format(k=' ' + i, lines=total, other=total - code, code=code)
317 s += hr
318 total = sum(self.total.values())
319 code = sum(self.code.values())
320 s += fmt.format(k='', lines=total, other=total - code, code=code)
321 print(s)
323 if self.excluded and verbose:
324 ex = fmt.format(k="Excluded", lines="Line", other="Other", code="Code")
325 ex += hr
326 for m in sorted(self.excluded):
327 for i in sorted(self.excluded[m], key=lambda i: self.excluded[m][i][0], reverse=True):
328 code, total = self.excluded[m][i]
329 ex += fmt.format(k=' ' + i, lines=total, other=total - code, code=code)
330 ex += hr
331 print(ex)
333 if self.errors:
334 e = "\nErrors\n\n"
335 for m in sorted(self.errors):
336 e += "{}\n".format(m)
337 for i in sorted(self.errors[m]):
338 e += fmt.format(k=' ' + i, lines=self.errors[m][i], other='', code='')
339 print(e)