Coverage for adhoc-cicd-odoo-odoo / odoo / service / db.py: 18%
351 statements
« prev ^ index » next coverage.py v7.13.4, created at 2026-03-09 18:15 +0000
1import base64
2import functools
3import json
4import logging
5import os
6import shutil
7import subprocess
8import tempfile
9import zipfile
11from contextlib import closing
12from datetime import datetime
13from xml.etree import ElementTree as ET
15import psycopg2
16from psycopg2.extensions import quote_ident
17from pytz import country_timezones
19import odoo.api
20import odoo.modules.neutralize
21import odoo.release
22import odoo.sql_db
23import odoo.tools
24from odoo.exceptions import AccessDenied
25from odoo.release import version_info
26from odoo.sql_db import db_connect
27from odoo.tools import osutil, SQL
28from odoo.tools.misc import exec_pg_environ, find_pg_tool
30_logger = logging.getLogger(__name__)
33class DatabaseExists(Warning):
34 pass
37def database_identifier(cr, name: str) -> SQL:
38 """Quote a database identifier.
40 Use instead of `SQL.identifier` to accept all kinds of identifiers.
41 """
42 name = quote_ident(name, cr._cnx)
43 return SQL(name)
46def check_db_management_enabled(func, /):
47 @functools.wraps(func)
48 def if_db_mgt_enabled(*args, **kwargs):
49 if not odoo.tools.config['list_db']:
50 _logger.error('Database management functions blocked, admin disabled database listing')
51 raise AccessDenied()
52 return func(*args, **kwargs)
53 return if_db_mgt_enabled
55# ----------------------------------------------------------
56# Master password required
57# ----------------------------------------------------------
60def check_super(passwd):
61 if passwd and odoo.tools.config.verify_admin_password(passwd):
62 return True
63 raise odoo.exceptions.AccessDenied()
66# This should be moved to odoo.modules.db, along side initialize().
67def _initialize_db(db_name, demo, lang, user_password, login='admin', country_code=None, phone=None):
68 try:
69 odoo.tools.config['load_language'] = lang
71 registry = odoo.modules.registry.Registry.new(db_name, update_module=True, new_db_demo=demo)
73 with closing(registry.cursor()) as cr:
74 env = odoo.api.Environment(cr, odoo.api.SUPERUSER_ID, {})
76 if lang:
77 modules = env['ir.module.module'].search([('state', '=', 'installed')])
78 modules._update_translations(lang)
80 if country_code:
81 country = env['res.country'].search([('code', 'ilike', country_code)])[0]
82 env['res.company'].browse(1).write({'country_id': country_code and country.id, 'currency_id': country_code and country.currency_id.id})
83 if len(country_timezones.get(country_code, [])) == 1:
84 users = env['res.users'].search([])
85 users.write({'tz': country_timezones[country_code][0]})
86 if phone:
87 env['res.company'].browse(1).write({'phone': phone})
88 if '@' in login:
89 env['res.company'].browse(1).write({'email': login})
91 # update admin's password and lang and login
92 values = {'password': user_password, 'lang': lang}
93 if login:
94 values['login'] = login
95 emails = odoo.tools.email_split(login)
96 if emails:
97 values['email'] = emails[0]
98 env.ref('base.user_admin').write(values)
100 cr.commit()
101 except Exception as e:
102 _logger.exception('CREATE DATABASE failed:')
105def _check_faketime_mode(db_name):
106 if os.getenv('ODOO_FAKETIME_TEST_MODE') and db_name in odoo.tools.config['db_name']: 106 ↛ 107line 106 didn't jump to line 107 because the condition on line 106 was never true
107 try:
108 db = odoo.sql_db.db_connect(db_name)
109 with db.cursor() as cursor:
110 cursor.execute("SELECT (pg_catalog.now() AT TIME ZONE 'UTC');")
111 server_now = cursor.fetchone()[0]
112 time_offset = (datetime.now() - server_now).total_seconds()
114 cursor.execute("""
115 CREATE OR REPLACE FUNCTION public.now()
116 RETURNS timestamp with time zone AS $$
117 SELECT pg_catalog.now() + %s * interval '1 second';
118 $$ LANGUAGE sql;
119 """, (int(time_offset), ))
120 cursor.execute("SELECT (now() AT TIME ZONE 'UTC');")
121 new_now = cursor.fetchone()[0]
122 _logger.info("Faketime mode, new cursor now is %s", new_now)
123 cursor.commit()
124 except psycopg2.Error as e:
125 _logger.warning("Unable to set fakedtimed NOW() : %s", e)
128def _create_empty_database(name):
129 db = odoo.sql_db.db_connect('postgres')
130 with closing(db.cursor()) as cr:
131 chosen_template = odoo.tools.config['db_template']
132 cr.execute("SELECT datname FROM pg_database WHERE datname = %s",
133 (name,), log_exceptions=False)
134 if cr.fetchall(): 134 ↛ 139line 134 didn't jump to line 139 because the condition on line 134 was always true
135 _check_faketime_mode(name)
136 raise DatabaseExists("database %r already exists!" % (name,))
137 else:
138 # database-altering operations cannot be executed inside a transaction
139 cr.rollback()
140 cr._cnx.autocommit = True
142 # 'C' collate is only safe with template0, but provides more useful indexes
143 cr.execute(SQL(
144 "CREATE DATABASE %s ENCODING 'unicode' %s TEMPLATE %s",
145 database_identifier(cr, name),
146 SQL("LC_COLLATE 'C'") if chosen_template == 'template0' else SQL(""),
147 database_identifier(cr, chosen_template),
148 ))
150 # TODO: add --extension=trigram,unaccent
151 try:
152 db = odoo.sql_db.db_connect(name)
153 with db.cursor() as cr:
154 cr.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm")
155 if odoo.tools.config['unaccent']:
156 cr.execute("CREATE EXTENSION IF NOT EXISTS unaccent")
157 # From PostgreSQL's point of view, making 'unaccent' immutable is incorrect
158 # because it depends on external data - see
159 # https://www.postgresql.org/message-id/flat/201012021544.oB2FiTn1041521@wwwmaster.postgresql.org#201012021544.oB2FiTn1041521@wwwmaster.postgresql.org
160 # But in the case of Odoo, we consider that those data don't
161 # change in the lifetime of a database. If they do change, all
162 # indexes created with this function become corrupted!
163 cr.execute("ALTER FUNCTION unaccent(text) IMMUTABLE")
164 except psycopg2.Error as e:
165 _logger.warning("Unable to create PostgreSQL extensions : %s", e)
166 _check_faketime_mode(name)
168 # restore legacy behaviour on pg15+
169 try:
170 db = odoo.sql_db.db_connect(name)
171 with db.cursor() as cr:
172 cr.execute("GRANT CREATE ON SCHEMA PUBLIC TO PUBLIC")
173 except psycopg2.Error as e:
174 _logger.warning("Unable to make public schema public-accessible: %s", e)
176@check_db_management_enabled
177def exp_create_database(db_name, demo, lang, user_password='admin', login='admin', country_code=None, phone=None):
178 """ Similar to exp_create but blocking."""
179 _logger.info('Create database `%s`.', db_name)
180 _create_empty_database(db_name)
181 _initialize_db(db_name, demo, lang, user_password, login, country_code, phone)
182 return True
184@check_db_management_enabled
185def exp_duplicate_database(db_original_name, db_name, neutralize_database=False):
186 _logger.info('Duplicate database `%s` to `%s`.', db_original_name, db_name)
187 odoo.sql_db.close_db(db_original_name)
188 db = odoo.sql_db.db_connect('postgres')
189 with closing(db.cursor()) as cr:
190 # database-altering operations cannot be executed inside a transaction
191 cr._cnx.autocommit = True
192 _drop_conn(cr, db_original_name)
193 cr.execute(SQL(
194 "CREATE DATABASE %s ENCODING 'unicode' TEMPLATE %s",
195 database_identifier(cr, db_name),
196 database_identifier(cr, db_original_name),
197 ))
199 registry = odoo.modules.registry.Registry.new(db_name)
200 with registry.cursor() as cr:
201 # if it's a copy of a database, force generation of a new dbuuid
202 env = odoo.api.Environment(cr, odoo.api.SUPERUSER_ID, {})
203 env['ir.config_parameter'].init(force=True)
204 if neutralize_database:
205 odoo.modules.neutralize.neutralize_database(cr)
207 from_fs = odoo.tools.config.filestore(db_original_name)
208 to_fs = odoo.tools.config.filestore(db_name)
209 if os.path.exists(from_fs) and not os.path.exists(to_fs):
210 shutil.copytree(from_fs, to_fs)
211 return True
213def _drop_conn(cr, db_name):
214 # Try to terminate all other connections that might prevent
215 # dropping the database
216 try:
217 # PostgreSQL 9.2 renamed pg_stat_activity.procpid to pid:
218 # http://www.postgresql.org/docs/9.2/static/release-9-2.html#AEN110389
219 pid_col = 'pid' if cr._cnx.server_version >= 90200 else 'procpid'
221 cr.execute("""SELECT pg_terminate_backend(%(pid_col)s)
222 FROM pg_stat_activity
223 WHERE datname = %%s AND
224 %(pid_col)s != pg_backend_pid()""" % {'pid_col': pid_col},
225 (db_name,))
226 except Exception:
227 pass
229@check_db_management_enabled
230def exp_drop(db_name):
231 if db_name not in list_dbs(True):
232 return False
233 odoo.modules.registry.Registry.delete(db_name)
234 odoo.sql_db.close_db(db_name)
236 db = odoo.sql_db.db_connect('postgres')
237 with closing(db.cursor()) as cr:
238 # database-altering operations cannot be executed inside a transaction
239 cr._cnx.autocommit = True
240 _drop_conn(cr, db_name)
242 try:
243 cr.execute(SQL('DROP DATABASE %s', database_identifier(cr, db_name)))
244 except Exception as e:
245 _logger.info('DROP DB: %s failed:\n%s', db_name, e)
246 raise Exception("Couldn't drop database %s: %s" % (db_name, e))
247 else:
248 _logger.info('DROP DB: %s', db_name)
250 fs = odoo.tools.config.filestore(db_name)
251 if os.path.exists(fs):
252 shutil.rmtree(fs)
253 return True
255@check_db_management_enabled
256def exp_dump(db_name, format):
257 with tempfile.TemporaryFile(mode='w+b') as t:
258 dump_db(db_name, t, format)
259 t.seek(0)
260 return base64.b64encode(t.read()).decode()
262@check_db_management_enabled
263def dump_db_manifest(cr):
264 pg_version = "%d.%d" % divmod(cr._obj.connection.server_version / 100, 100)
265 cr.execute("SELECT name, latest_version FROM ir_module_module WHERE state = 'installed'")
266 modules = dict(cr.fetchall())
267 manifest = {
268 'odoo_dump': '1',
269 'db_name': cr.dbname,
270 'version': odoo.release.version,
271 'version_info': odoo.release.version_info,
272 'major_version': odoo.release.major_version,
273 'pg_version': pg_version,
274 'modules': modules,
275 }
276 return manifest
278@check_db_management_enabled
279def dump_db(db_name, stream, backup_format='zip', with_filestore=True):
280 """Dump database `db` into file-like object `stream` if stream is None
281 return a file object with the dump """
283 _logger.info('DUMP DB: %s format %s %s', db_name, backup_format, 'with filestore' if with_filestore else 'without filestore')
285 cmd = [find_pg_tool('pg_dump'), '--no-owner', db_name]
286 env = exec_pg_environ()
288 if backup_format == 'zip':
289 with tempfile.TemporaryDirectory() as dump_dir:
290 if with_filestore:
291 filestore = odoo.tools.config.filestore(db_name)
292 if os.path.exists(filestore):
293 shutil.copytree(filestore, os.path.join(dump_dir, 'filestore'))
294 with open(os.path.join(dump_dir, 'manifest.json'), 'w') as fh:
295 db = odoo.sql_db.db_connect(db_name)
296 with db.cursor() as cr:
297 json.dump(dump_db_manifest(cr), fh, indent=4)
298 cmd.insert(-1, '--file=' + os.path.join(dump_dir, 'dump.sql'))
299 subprocess.run(cmd, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT, check=True)
300 if stream:
301 osutil.zip_dir(dump_dir, stream, include_dir=False, fnct_sort=lambda file_name: file_name != 'dump.sql')
302 else:
303 t=tempfile.TemporaryFile()
304 osutil.zip_dir(dump_dir, t, include_dir=False, fnct_sort=lambda file_name: file_name != 'dump.sql')
305 t.seek(0)
306 return t
307 else:
308 cmd.insert(-1, '--format=c')
309 stdout = subprocess.Popen(cmd, env=env, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE).stdout
310 if stream:
311 shutil.copyfileobj(stdout, stream)
312 else:
313 return stdout
315@check_db_management_enabled
316def exp_restore(db_name, data, copy=False):
317 def chunks(d, n=8192):
318 for i in range(0, len(d), n):
319 yield d[i:i+n]
320 data_file = tempfile.NamedTemporaryFile(delete=False)
321 try:
322 for chunk in chunks(data):
323 data_file.write(base64.b64decode(chunk))
324 data_file.close()
325 restore_db(db_name, data_file.name, copy=copy)
326 finally:
327 os.unlink(data_file.name)
328 return True
330@check_db_management_enabled
331def restore_db(db, dump_file, copy=False, neutralize_database=False):
332 assert isinstance(db, str)
333 if exp_db_exist(db):
334 _logger.warning('RESTORE DB: %s already exists', db)
335 raise Exception("Database already exists")
337 _logger.info('RESTORING DB: %s', db)
338 _create_empty_database(db)
340 filestore_path = None
341 with tempfile.TemporaryDirectory() as dump_dir:
342 if zipfile.is_zipfile(dump_file):
343 # v8 format
344 with zipfile.ZipFile(dump_file, 'r') as z:
345 # only extract known members!
346 filestore = [m for m in z.namelist() if m.startswith('filestore/')]
347 z.extractall(dump_dir, ['dump.sql'] + filestore)
349 if filestore:
350 filestore_path = os.path.join(dump_dir, 'filestore')
352 pg_cmd = 'psql'
353 pg_args = ['-q', '-f', os.path.join(dump_dir, 'dump.sql')]
355 else:
356 # <= 7.0 format (raw pg_dump output)
357 pg_cmd = 'pg_restore'
358 pg_args = ['--no-owner', dump_file]
360 r = subprocess.run(
361 [find_pg_tool(pg_cmd), '--dbname=' + db, *pg_args],
362 env=exec_pg_environ(),
363 stdout=subprocess.DEVNULL,
364 stderr=subprocess.STDOUT,
365 )
366 if r.returncode != 0:
367 raise Exception("Couldn't restore database")
369 registry = odoo.modules.registry.Registry.new(db)
370 with registry.cursor() as cr:
371 env = odoo.api.Environment(cr, odoo.api.SUPERUSER_ID, {})
372 if copy:
373 # if it's a copy of a database, force generation of a new dbuuid
374 env['ir.config_parameter'].init(force=True)
375 if neutralize_database:
376 odoo.modules.neutralize.neutralize_database(cr)
378 if filestore_path:
379 filestore_dest = env['ir.attachment']._filestore()
380 shutil.move(filestore_path, filestore_dest)
382 _logger.info('RESTORE DB: %s', db)
384@check_db_management_enabled
385def exp_rename(old_name, new_name):
386 odoo.modules.registry.Registry.delete(old_name)
387 odoo.sql_db.close_db(old_name)
389 db = odoo.sql_db.db_connect('postgres')
390 with closing(db.cursor()) as cr:
391 # database-altering operations cannot be executed inside a transaction
392 cr._cnx.autocommit = True
393 _drop_conn(cr, old_name)
394 try:
395 cr.execute(SQL('ALTER DATABASE %s RENAME TO %s', database_identifier(cr, old_name), database_identifier(cr, new_name)))
396 _logger.info('RENAME DB: %s -> %s', old_name, new_name)
397 except Exception as e:
398 _logger.info('RENAME DB: %s -> %s failed:\n%s', old_name, new_name, e)
399 raise Exception("Couldn't rename database %s to %s: %s" % (old_name, new_name, e))
401 old_fs = odoo.tools.config.filestore(old_name)
402 new_fs = odoo.tools.config.filestore(new_name)
403 if os.path.exists(old_fs) and not os.path.exists(new_fs):
404 shutil.move(old_fs, new_fs)
405 return True
407@check_db_management_enabled
408def exp_change_admin_password(new_password):
409 odoo.tools.config.set_admin_password(new_password)
410 odoo.tools.config.save(['admin_passwd'])
411 return True
413@check_db_management_enabled
414def exp_migrate_databases(databases):
415 for db in databases:
416 _logger.info('migrate database %s', db)
417 odoo.modules.registry.Registry.new(db, update_module=True, upgrade_modules={'base'})
418 return True
420#----------------------------------------------------------
421# No master password required
422#----------------------------------------------------------
424@odoo.tools.mute_logger('odoo.sql_db')
425def exp_db_exist(db_name):
426 ## Not True: in fact, check if connection to database is possible. The database may exists
427 try:
428 db = odoo.sql_db.db_connect(db_name)
429 with db.cursor():
430 return True
431 except Exception:
432 return False
434def list_dbs(force=False):
435 if not odoo.tools.config['list_db'] and not force:
436 raise odoo.exceptions.AccessDenied()
438 if not odoo.tools.config['dbfilter'] and odoo.tools.config['db_name']:
439 # In case --db-filter is not provided and --database is passed, Odoo will not
440 # fetch the list of databases available on the postgres server and instead will
441 # use the value of --database as comma seperated list of exposed databases.
442 return sorted(odoo.tools.config['db_name'])
444 chosen_template = odoo.tools.config['db_template']
445 templates_list = tuple({'postgres', chosen_template})
446 db = odoo.sql_db.db_connect('postgres')
447 with closing(db.cursor()) as cr:
448 try:
449 cr.execute("select datname from pg_database where datdba=(select usesysid from pg_user where usename=current_user) and not datistemplate and datallowconn and datname not in %s order by datname", (templates_list,))
450 return [name for (name,) in cr.fetchall()]
451 except Exception:
452 _logger.exception('Listing databases failed:')
453 return []
455def list_db_incompatible(databases):
456 """"Check a list of databases if they are compatible with this version of Odoo
458 :param databases: A list of existing Postgresql databases
459 :return: A list of databases that are incompatible
460 """
461 incompatible_databases = []
462 server_version = '.'.join(str(v) for v in version_info[:2])
463 for database_name in databases:
464 with closing(db_connect(database_name).cursor()) as cr:
465 if odoo.tools.sql.table_exists(cr, 'ir_module_module'):
466 cr.execute("SELECT latest_version FROM ir_module_module WHERE name=%s", ('base',))
467 base_version = cr.fetchone()
468 if not base_version or not base_version[0]:
469 incompatible_databases.append(database_name)
470 else:
471 # e.g. 10.saas~15
472 local_version = '.'.join(base_version[0].split('.')[:2])
473 if local_version != server_version:
474 incompatible_databases.append(database_name)
475 else:
476 incompatible_databases.append(database_name)
477 for database_name in incompatible_databases:
478 # release connection
479 odoo.sql_db.close_db(database_name)
480 return incompatible_databases
483def exp_list(document=False):
484 if not odoo.tools.config['list_db']:
485 raise odoo.exceptions.AccessDenied()
486 return list_dbs()
488def exp_list_lang():
489 return odoo.tools.misc.scan_languages()
491def exp_list_countries():
492 list_countries = []
493 root = ET.parse(os.path.join(odoo.tools.config.root_path, 'addons/base/data/res_country_data.xml')).getroot()
494 for country in root.find('data').findall('record[@model="res.country"]'):
495 name = country.find('field[@name="name"]').text
496 code = country.find('field[@name="code"]').text
497 list_countries.append([code, name])
498 return sorted(list_countries, key=lambda c: c[1])
500def exp_server_version():
501 """ Return the version of the server
502 Used by the client to verify the compatibility with its own version
503 """
504 return odoo.release.version
506#----------------------------------------------------------
507# db service dispatch
508#----------------------------------------------------------
510def dispatch(method, params):
511 g = globals()
512 exp_method_name = 'exp_' + method
513 if method in ['db_exist', 'list', 'list_lang', 'server_version']:
514 return g[exp_method_name](*params)
515 elif exp_method_name in g:
516 passwd = params[0]
517 params = params[1:]
518 check_super(passwd)
519 return g[exp_method_name](*params)
520 else:
521 raise KeyError("Method not found: %s" % method)