Coverage for adhoc-cicd-odoo-odoo / odoo / modules / module.py: 62%
352 statements
« prev ^ index » next coverage.py v7.13.4, created at 2026-03-09 18:22 +0000
1# Part of Odoo. See LICENSE file for full copyright and licensing details.
2"""Utility functions to manage module manifest files and discovery."""
3from __future__ import annotations
5import ast
6import copy
7import functools
8import importlib
9import importlib.metadata
10import logging
11import os
12import re
13import sys
14import traceback
15import typing
16import warnings
17from collections.abc import Collection, Iterable, Mapping
18from os.path import join as opj
20import odoo.addons
21import odoo.release as release
22import odoo.tools as tools
23import odoo.upgrade
25try:
26 from packaging.requirements import InvalidRequirement, Requirement
27except ImportError:
28 class InvalidRequirement(Exception): # type: ignore[no-redef]
29 ...
31 class Requirement: # type: ignore[no-redef]
32 def __init__(self, pydep):
33 if not re.fullmatch(r'[\w\-]+', pydep): # check that we have no versions or marker in pydep
34 msg = f"Package `packaging` is required to parse `{pydep}` external dependency and is not installed"
35 raise Exception(msg)
36 self.marker = None
37 self.specifier = None
38 self.name = pydep
40__all__ = [
41 "Manifest",
42 "adapt_version",
43 "get_manifest",
44 "get_module_path",
45 "get_modules",
46 "get_modules_with_version",
47 "get_resource_from_path",
48 "initialize_sys_path",
49 "load_openerp_module",
50]
52MODULE_NAME_RE = re.compile(r'^\w{1,256}$')
53MANIFEST_NAMES = ['__manifest__.py']
54README = ['README.rst', 'README.md', 'README.txt', 'README']
56_DEFAULT_MANIFEST = {
57 # Mandatory fields (with no defaults):
58 # - author
59 # - license
60 # - name
61 # Derived fields are computed in the Manifest class.
62 'application': False,
63 'bootstrap': False, # web
64 'assets': {},
65 'auto_install': False,
66 'category': 'Uncategorized',
67 'cloc_exclude': [],
68 'configurator_snippets': {}, # website themes
69 'configurator_snippets_addons': {}, # website themes
70 'countries': [],
71 'data': [],
72 'demo': [],
73 'demo_xml': [],
74 'depends': [],
75 'description': '', # defaults to README file
76 'external_dependencies': {},
77 'init_xml': [],
78 'installable': True,
79 'images': [], # website
80 'images_preview_theme': {}, # website themes
81 'live_test_url': '', # website themes
82 'new_page_templates': {}, # website themes
83 'post_init_hook': '',
84 'post_load': '',
85 'pre_init_hook': '',
86 'sequence': 100,
87 'summary': '',
88 'test': [],
89 'theme_customizations': {}, # themes
90 'update_xml': [],
91 'uninstall_hook': '',
92 'version': '1.0',
93 'web': False,
94 'website': '',
95}
97# matches field definitions like
98# partner_id: base.ResPartner = fields.Many2one
99# partner_id = fields.Many2one[base.ResPartner]
100TYPED_FIELD_DEFINITION_RE = re.compile(r'''
101 \b (?P<field_name>\w+) \s*
102 (:\s*(?P<field_type>[^ ]*))? \s*
103 = \s*
104 fields\.(?P<field_class>Many2one|One2many|Many2many)
105 (\[(?P<type_param>[^\]]+)\])?
106''', re.VERBOSE)
108_logger = logging.getLogger(__name__)
110current_test: bool = False
111"""Indicates whteher we are in a test mode"""
114class UpgradeHook:
115 """Makes the legacy `migrations` package being `odoo.upgrade`"""
117 def find_spec(self, fullname, path=None, target=None):
118 if re.match(r"^odoo\.addons\.base\.maintenance\.migrations\b", fullname): 118 ↛ 123line 118 didn't jump to line 123 because the condition on line 118 was never true
119 # We can't trigger a DeprecationWarning in this case.
120 # In order to be cross-versions, the multi-versions upgrade scripts (0.0.0 scripts),
121 # the tests, and the common files (utility functions) still needs to import from the
122 # legacy name.
123 return importlib.util.spec_from_loader(fullname, self)
125 def load_module(self, name):
126 assert name not in sys.modules
128 canonical_upgrade = name.replace("odoo.addons.base.maintenance.migrations", "odoo.upgrade")
130 if canonical_upgrade in sys.modules:
131 mod = sys.modules[canonical_upgrade]
132 else:
133 mod = importlib.import_module(canonical_upgrade)
135 sys.modules[name] = mod
137 return sys.modules[name]
140def initialize_sys_path() -> None:
141 """
142 Setup the addons path ``odoo.addons.__path__`` with various defaults
143 and explicit directories.
144 """
145 for path in (
146 # tools.config.addons_base_dir, # already present
147 tools.config.addons_data_dir,
148 *tools.config['addons_path'],
149 tools.config.addons_community_dir,
150 ):
151 if os.access(path, os.R_OK) and path not in odoo.addons.__path__:
152 odoo.addons.__path__.append(path)
154 # hook odoo.upgrade on upgrade-path
155 legacy_upgrade_path = os.path.join(tools.config.addons_base_dir, 'base/maintenance/migrations')
156 for up in tools.config['upgrade_path'] or [legacy_upgrade_path]:
157 if up not in odoo.upgrade.__path__:
158 odoo.upgrade.__path__.append(up)
160 # create decrecated module alias from odoo.addons.base.maintenance.migrations to odoo.upgrade
161 spec = importlib.machinery.ModuleSpec("odoo.addons.base.maintenance", None, is_package=True)
162 maintenance_pkg = importlib.util.module_from_spec(spec)
163 maintenance_pkg.migrations = odoo.upgrade # type: ignore
164 sys.modules["odoo.addons.base.maintenance"] = maintenance_pkg
165 sys.modules["odoo.addons.base.maintenance.migrations"] = odoo.upgrade
167 # hook for upgrades and namespace freeze
168 if not getattr(initialize_sys_path, 'called', False): # only initialize once
169 odoo.addons.__path__._path_finder = lambda *a: None # prevent path invalidation
170 odoo.upgrade.__path__._path_finder = lambda *a: None # prevent path invalidation
171 sys.meta_path.insert(0, UpgradeHook())
172 initialize_sys_path.called = True # type: ignore
175@typing.final
176class Manifest(Mapping[str, typing.Any]):
177 """The manifest data of a module."""
179 def __init__(self, *, path: str, manifest_content: dict):
180 assert os.path.isabs(path), "path of module must be absolute"
181 self.path = path
182 _, self.name = os.path.split(path)
183 if not MODULE_NAME_RE.match(self.name): 183 ↛ 184line 183 didn't jump to line 184 because the condition on line 183 was never true
184 raise FileNotFoundError(f"Invalid module name: {self.name}")
185 self.__manifest_content = manifest_content
187 @property
188 def addons_path(self) -> str:
189 parent_path, name = os.path.split(self.path)
190 assert name == self.name
191 return parent_path
193 @functools.cached_property
194 def __manifest_cached(self) -> dict:
195 """Parsed and validated manifest data from the file."""
196 return _load_manifest(self.name, self.__manifest_content)
198 @functools.cached_property
199 def description(self):
200 """The description of the module defaulting to the README file."""
201 if (desc := self.__manifest_cached.get('description')):
202 return desc
203 for file_name in README:
204 try:
205 with tools.file_open(opj(self.path, file_name)) as f:
206 return f.read()
207 except OSError:
208 pass
209 return ''
211 @functools.cached_property
212 def version(self):
213 try:
214 return self.__manifest_cached['version']
215 except Exception: # noqa: BLE001
216 return adapt_version('1.0')
218 @functools.cached_property
219 def icon(self) -> str:
220 return get_module_icon(self.name)
222 @functools.cached_property
223 def static_path(self) -> str | None:
224 static_path = opj(self.path, 'static')
225 manifest = self.__manifest_cached
226 if (manifest['installable'] or manifest['assets']) and os.path.isdir(static_path):
227 return static_path
228 return None
230 def __getitem__(self, key: str):
231 if key in ('description', 'icon', 'addons_path', 'version', 'static_path'):
232 return getattr(self, key)
233 return copy.deepcopy(self.__manifest_cached[key])
235 def raw_value(self, key):
236 return copy.deepcopy(self.__manifest_cached.get(key))
238 def __iter__(self):
239 manifest = self.__manifest_cached
240 yield from manifest
241 for key in ('description', 'icon', 'addons_path', 'version', 'static_path'):
242 if key not in manifest:
243 yield key
245 def check_manifest_dependencies(self) -> None:
246 """Check that the dependecies of the manifest are available.
248 - Checking for external python dependencies
249 - Checking binaries are available in PATH
251 On missing dependencies, raise an error.
252 """
253 depends = self.get('external_dependencies')
254 if not depends:
255 return
256 for pydep in depends.get('python', []):
257 check_python_external_dependency(pydep)
259 for binary in depends.get('bin', []): 259 ↛ 260line 259 didn't jump to line 260 because the loop on line 259 never started
260 try:
261 tools.find_in_path(binary)
262 except OSError:
263 msg = "Unable to find {dependency!r} in path"
264 raise MissingDependency(msg, binary)
266 def __bool__(self):
267 return True
269 def __len__(self):
270 return sum(1 for _ in self)
272 def __repr__(self):
273 return f'Manifest({self.name})'
275 # limit cache size because this may get called from any module with any input
276 @staticmethod
277 @functools.lru_cache(10_000)
278 def _get_manifest_from_addons(module: str) -> Manifest | None:
279 """Get the module's manifest from a name. Searching only in addons paths."""
280 for adp in odoo.addons.__path__: 280 ↛ 283line 280 didn't jump to line 283 because the loop on line 280 didn't complete
281 if manifest := Manifest._from_path(opj(adp, module)):
282 return manifest
283 return None
285 @staticmethod
286 def for_addon(module_name: str, *, display_warning: bool = True) -> Manifest | None:
287 """Get the module's manifest from a name.
289 :param module: module's name
290 :param display_warning: log a warning if the module is not found
291 """
292 if not MODULE_NAME_RE.match(module_name): 292 ↛ 294line 292 didn't jump to line 294 because the condition on line 292 was never true
293 # invalid module name
294 return None
295 if mod := Manifest._get_manifest_from_addons(module_name): 295 ↛ 297line 295 didn't jump to line 297 because the condition on line 295 was always true
296 return mod
297 if display_warning:
298 _logger.warning('module %s: manifest not found', module_name)
299 return None
301 @staticmethod
302 def _from_path(path: str, env=None) -> Manifest | None:
303 """Given a path, read the manifest file."""
304 for manifest_name in MANIFEST_NAMES:
305 try:
306 with tools.file_open(opj(path, manifest_name), env=env) as f:
307 manifest_content = ast.literal_eval(f.read())
308 except OSError:
309 pass
310 except Exception: # noqa: BLE001
311 _logger.debug("Failed to parse the manifest file at %r", path, exc_info=True)
312 else:
313 return Manifest(path=path, manifest_content=manifest_content)
314 return None
316 @staticmethod
317 def all_addon_manifests() -> list[Manifest]:
318 """Read all manifests in the addons paths."""
319 modules: dict[str, Manifest] = {}
320 for adp in odoo.addons.__path__:
321 if not os.path.isdir(adp): 321 ↛ 322line 321 didn't jump to line 322 because the condition on line 321 was never true
322 _logger.warning("addons path is not a directory: %s", adp)
323 continue
324 for file_name in os.listdir(adp):
325 if file_name in modules: 325 ↛ 326line 325 didn't jump to line 326 because the condition on line 325 was never true
326 continue
327 if mod := Manifest._from_path(opj(adp, file_name)):
328 assert file_name == mod.name
329 modules[file_name] = mod
330 return sorted(modules.values(), key=lambda m: m.name)
333def get_module_path(module: str, display_warning: bool = True) -> str | None:
334 """Return the path of the given module.
336 Search the addons paths and return the first path where the given
337 module is found. If downloaded is True, return the default addons
338 path if nothing else is found.
340 """
341 # TODO deprecate
342 mod = Manifest.for_addon(module, display_warning=display_warning)
343 return mod.path if mod else None
346def get_resource_from_path(path: str) -> tuple[str, str, str] | None:
347 """Tries to extract the module name and the resource's relative path
348 out of an absolute resource path.
350 If operation is successful, returns a tuple containing the module name, the relative path
351 to the resource using '/' as filesystem seperator[1] and the same relative path using
352 os.path.sep seperators.
354 [1] same convention as the resource path declaration in manifests
356 :param path: absolute resource path
358 :rtype: tuple
359 :return: tuple(module_name, relative_path, os_relative_path) if possible, else None
360 """
361 resource = None
362 sorted_paths = sorted(odoo.addons.__path__, key=len, reverse=True)
363 for adpath in sorted_paths: 363 ↛ 370line 363 didn't jump to line 370 because the loop on line 363 didn't complete
364 # force trailing separator
365 adpath = os.path.join(adpath, "")
366 if os.path.commonprefix([adpath, path]) == adpath:
367 resource = path.replace(adpath, "", 1)
368 break
370 if resource: 370 ↛ 376line 370 didn't jump to line 376 because the condition on line 370 was always true
371 relative = resource.split(os.path.sep)
372 if not relative[0]: 372 ↛ 373line 372 didn't jump to line 373 because the condition on line 372 was never true
373 relative.pop(0)
374 module = relative.pop(0)
375 return (module, '/'.join(relative), os.path.sep.join(relative))
376 return None
379def get_module_icon(module: str) -> str:
380 """ Get the path to the module's icon. Invalid module names are accepted. """
381 manifest = Manifest.for_addon(module, display_warning=False)
382 if manifest and 'icon' in manifest.__dict__: 382 ↛ 384line 382 didn't jump to line 384 because the condition on line 382 was never true
383 # we have a value in the cached property
384 return manifest.icon
385 fpath = ''
386 if manifest: 386 ↛ 389line 386 didn't jump to line 389 because the condition on line 386 was always true
387 fpath = manifest.raw_value('icon') or ''
388 fpath = fpath.lstrip('/')
389 if not fpath:
390 fpath = f"{module}/static/description/icon.png"
391 try:
392 tools.file_path(fpath)
393 return "/" + fpath
394 except FileNotFoundError:
395 return "/base/static/description/icon.png"
398def load_manifest(module: str, mod_path: str | None = None) -> dict:
399 """ Load the module manifest from the file system. """
400 warnings.warn("Since 19.0, use Manifest", DeprecationWarning)
402 if mod_path:
403 mod = Manifest._from_path(mod_path)
404 assert mod.path == mod_path
405 else:
406 mod = Manifest.for_addon(module)
407 if not mod:
408 _logger.debug('module %s: no manifest file found %s', module, MANIFEST_NAMES)
409 return {}
411 return dict(mod)
414def _load_manifest(module: str, manifest_content: dict) -> dict:
415 """ Load and validate the module manifest.
417 Return a new dictionary with cleaned and validated keys.
418 """
420 manifest = copy.deepcopy(_DEFAULT_MANIFEST)
421 manifest.update(manifest_content)
423 if not manifest.get('author'): 423 ↛ 427line 423 didn't jump to line 427 because the condition on line 423 was never true
424 # Altought contributors and maintainer are not documented, it is
425 # not uncommon to find them in manifest files, use them as
426 # alternative.
427 author = manifest.get('contributors') or manifest.get('maintainer') or ''
428 manifest['author'] = str(author)
429 _logger.warning("Missing `author` key in manifest for %r, defaulting to %r", module, str(author))
431 if not manifest.get('license'): 431 ↛ 432line 431 didn't jump to line 432 because the condition on line 431 was never true
432 manifest['license'] = 'LGPL-3'
433 _logger.warning("Missing `license` key in manifest for %r, defaulting to LGPL-3", module)
435 if module == 'base':
436 manifest['depends'] = []
437 elif not manifest['depends']:
438 # prevent the hack `'depends': []` except 'base' module
439 manifest['depends'] = ['base']
441 depends = manifest['depends']
442 assert isinstance(depends, Collection)
444 # auto_install is either `False` (by default) in which case the module
445 # is opt-in, either a list of dependencies in which case the module is
446 # automatically installed if all dependencies are (special case: [] to
447 # always install the module), either `True` to auto-install the module
448 # in case all dependencies declared in `depends` are installed.
449 if isinstance(manifest['auto_install'], Iterable):
450 manifest['auto_install'] = auto_install_set = set(manifest['auto_install'])
451 non_dependencies = auto_install_set.difference(depends)
452 assert not non_dependencies, (
453 "auto_install triggers must be dependencies,"
454 f" found non-dependencies [{', '.join(non_dependencies)}] for module {module}"
455 )
456 elif manifest['auto_install']:
457 manifest['auto_install'] = set(depends)
459 try:
460 manifest['version'] = adapt_version(str(manifest['version']))
461 except ValueError as e:
462 if manifest['installable']:
463 raise ValueError(f"Module {module}: invalid manifest") from e
464 if manifest['installable'] and not check_version(str(manifest['version']), should_raise=False): 464 ↛ 465line 464 didn't jump to line 465 because the condition on line 464 was never true
465 _logger.warning("The module %s has an incompatible version, setting installable=False", module)
466 manifest['installable'] = False
468 return manifest
471def get_manifest(module: str, mod_path: str | None = None) -> Mapping[str, typing.Any]:
472 """
473 Get the module manifest.
475 :param str module: The name of the module (sale, purchase, ...).
476 :param Optional[str] mod_path: The optional path to the module on
477 the file-system. If not set, it is determined by scanning the
478 addons-paths.
479 :returns: The module manifest as a dict or an empty dict
480 when the manifest was not found.
481 """
482 if mod_path:
483 mod = Manifest._from_path(mod_path)
484 if mod and mod.name != module:
485 raise ValueError(f"Invalid path for module {module}: {mod_path}")
486 else:
487 mod = Manifest.for_addon(module, display_warning=False)
488 return mod if mod is not None else {}
491def load_openerp_module(module_name: str) -> None:
492 """ Load an OpenERP module, if not already loaded.
494 This loads the module and register all of its models, thanks to either
495 the MetaModel metaclass, or the explicit instantiation of the model.
496 This is also used to load server-wide module (i.e. it is also used
497 when there is no model to register).
498 """
500 qualname = f'odoo.addons.{module_name}'
501 if qualname in sys.modules:
502 return
504 try:
505 __import__(qualname)
507 # Call the module's post-load hook. This can done before any model or
508 # data has been initialized. This is ok as the post-load hook is for
509 # server-wide (instead of registry-specific) functionalities.
510 manifest = Manifest.for_addon(module_name)
511 if post_load := manifest.get('post_load'):
512 getattr(sys.modules[qualname], post_load)()
514 except AttributeError as err:
515 _logger.critical("Couldn't load module %s", module_name)
516 trace = traceback.format_exc()
517 match = TYPED_FIELD_DEFINITION_RE.search(trace)
518 if match and "most likely due to a circular import" in trace:
519 field_name = match['field_name']
520 field_class = match['field_class']
521 field_type = match['field_type'] or match['type_param']
522 if "." not in field_type:
523 field_type = f"{module_name}.{field_type}"
524 raise AttributeError(
525 f"{err}\n"
526 "To avoid circular import for the the comodel use the annotation syntax:\n"
527 f" {field_name}: {field_type} = fields.{field_class}(...)\n"
528 "and add at the beggining of the file:\n"
529 " from __future__ import annotations"
530 ).with_traceback(err.__traceback__) from None
531 raise
532 except Exception:
533 _logger.critical("Couldn't load module %s", module_name)
534 raise
537def get_modules() -> list[str]:
538 """Get the list of module names that can be loaded.
539 """
540 return [m.name for m in Manifest.all_addon_manifests()]
543def get_modules_with_version() -> dict[str, str]:
544 """Get the module list with the linked version."""
545 warnings.warn("Since 19.0, use Manifest.all_addon_manifests", DeprecationWarning)
546 return {m.name: m.version for m in Manifest.all_addon_manifests()}
549def adapt_version(version: str) -> str:
550 """Reformat the version of the module into a canonical format."""
551 version_str_parts = version.split('.')
552 if not (2 <= len(version_str_parts) <= 5): 552 ↛ 553line 552 didn't jump to line 553 because the condition on line 552 was never true
553 raise ValueError(f"Invalid version {version!r}, must have between 2 and 5 parts")
554 serie = release.major_version
555 if version.startswith(serie) and not version_str_parts[0].isdigit(): 555 ↛ 557line 555 didn't jump to line 557 because the condition on line 555 was never true
556 # keep only digits for parsing
557 version_str_parts[0] = ''.join(c for c in version_str_parts[0] if c.isdigit())
558 try:
559 version_parts = [int(v) for v in version_str_parts]
560 except ValueError as e:
561 raise ValueError(f"Invalid version {version!r}") from e
562 if len(version_parts) <= 3 and not version.startswith(serie):
563 # prefix the version with serie
564 return f"{serie}.{version}"
565 return version
568def check_version(version: str, should_raise: bool = True) -> bool:
569 """Check that the version is in a valid format for the current release."""
570 version = adapt_version(version)
571 serie = release.major_version
572 if version.startswith(serie + '.'): 572 ↛ 574line 572 didn't jump to line 574 because the condition on line 572 was always true
573 return True
574 if should_raise:
575 raise ValueError(
576 f"Invalid version {version!r}. Modules should have a version in format"
577 f" `x.y`, `x.y.z`, `{serie}.x.y` or `{serie}.x.y.z`.")
578 return False
581class MissingDependency(Exception):
582 def __init__(self, msg_template: str, dependency: str):
583 self.dependency = dependency
584 super().__init__(msg_template.format(dependency=dependency))
587def check_python_external_dependency(pydep: str) -> None:
588 try:
589 requirement = Requirement(pydep)
590 except InvalidRequirement as e:
591 msg = f"{pydep} is an invalid external dependency specification: {e}"
592 raise ValueError(msg) from e
593 if requirement.marker and not requirement.marker.evaluate(): 593 ↛ 594line 593 didn't jump to line 594 because the condition on line 593 was never true
594 _logger.debug(
595 "Ignored external dependency %s because environment markers do not match",
596 pydep
597 )
598 return
599 try:
600 version = importlib.metadata.version(requirement.name)
601 except importlib.metadata.PackageNotFoundError as e:
602 try:
603 # keep compatibility with module name but log a warning instead of info
604 importlib.import_module(pydep)
605 _logger.warning("python external dependency on '%s' does not appear o be a valid PyPI package. Using a PyPI package name is recommended.", pydep)
606 return
607 except ImportError:
608 pass
609 msg = "External dependency {dependency!r} not installed: %s" % (e,)
610 raise MissingDependency(msg, pydep) from e
611 if requirement.specifier and not requirement.specifier.contains(version): 611 ↛ 612line 611 didn't jump to line 612 because the condition on line 611 was never true
612 msg = f"External dependency version mismatch: {{dependency}} (installed: {version})"
613 raise MissingDependency(msg, pydep)
616def load_script(path: str, module_name: str):
617 full_path = tools.file_path(path) if not os.path.isabs(path) else path
618 spec = importlib.util.spec_from_file_location(module_name, full_path)
619 assert spec and spec.loader, f"spec not found for {module_name}"
620 module = importlib.util.module_from_spec(spec)
621 spec.loader.exec_module(module)
622 return module