Coverage for mindsdb / api / http / initialize.py: 50%
298 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-01-21 00:36 +0000
1import os
2import secrets
3import mimetypes
4import threading
5import webbrowser
7from pathlib import Path
8from http import HTTPStatus
11import requests
12from flask import Flask, url_for, request, send_from_directory
13from flask_compress import Compress
14from flask_restx import Api
15from werkzeug.exceptions import HTTPException
16from packaging.version import Version, parse as parse_version
18from mindsdb.__about__ import __version__ as mindsdb_version
19from mindsdb.api.http.gui import update_static
20from mindsdb.api.http.utils import http_error
21from mindsdb.api.http.namespaces.agents import ns_conf as agents_ns
22from mindsdb.api.http.namespaces.analysis import ns_conf as analysis_ns
23from mindsdb.api.http.namespaces.auth import ns_conf as auth_ns
24from mindsdb.api.http.namespaces.chatbots import ns_conf as chatbots_ns
25from mindsdb.api.http.namespaces.jobs import ns_conf as jobs_ns
26from mindsdb.api.http.namespaces.config import ns_conf as conf_ns
27from mindsdb.api.http.namespaces.databases import ns_conf as databases_ns
28from mindsdb.api.http.namespaces.default import ns_conf as default_ns, check_session_auth
29from mindsdb.api.http.namespaces.file import ns_conf as file_ns
30from mindsdb.api.http.namespaces.handlers import ns_conf as handlers_ns
31from mindsdb.api.http.namespaces.knowledge_bases import ns_conf as knowledge_bases_ns
32from mindsdb.api.http.namespaces.models import ns_conf as models_ns
33from mindsdb.api.http.namespaces.projects import ns_conf as projects_ns
34from mindsdb.api.http.namespaces.skills import ns_conf as skills_ns
35from mindsdb.api.http.namespaces.sql import ns_conf as sql_ns
36from mindsdb.api.http.namespaces.tab import ns_conf as tab_ns
37from mindsdb.api.http.namespaces.tree import ns_conf as tree_ns
38from mindsdb.api.http.namespaces.views import ns_conf as views_ns
39from mindsdb.api.http.namespaces.util import ns_conf as utils_ns
40from mindsdb.api.http.namespaces.webhooks import ns_conf as webhooks_ns
41from mindsdb.interfaces.database.integrations import integration_controller
42from mindsdb.interfaces.database.database import DatabaseController
43from mindsdb.interfaces.file.file_controller import FileController
44from mindsdb.interfaces.jobs.jobs_controller import JobsController
45from mindsdb.interfaces.storage import db
46from mindsdb.metrics.server import init_metrics
47from mindsdb.utilities import log
48from mindsdb.utilities.config import config, HTTP_AUTH_TYPE
49from mindsdb.utilities.context import context as ctx
50from mindsdb.utilities.json_encoder import ORJSONProvider
51from mindsdb.utilities.ps import is_pid_listen_port, wait_func_is_true
52from mindsdb.utilities.sentry import sentry_sdk # noqa: F401
53from mindsdb.utilities.otel import trace # noqa: F401
54from mindsdb.api.common.middleware import verify_pat
56logger = log.getLogger(__name__)
59class _NoOpFlaskInstrumentor:
60 def instrument_app(self, app):
61 pass
64class _NoOpRequestsInstrumentor:
65 def instrument(self):
66 pass
69try:
70 from opentelemetry.instrumentation.flask import FlaskInstrumentor
71 from opentelemetry.instrumentation.requests import RequestsInstrumentor
72except ImportError:
73 logger.debug(
74 "OpenTelemetry is not avaiable. Please run `pip install -r requirements/requirements-opentelemetry.txt` to use it."
75 )
76 FlaskInstrumentor = _NoOpFlaskInstrumentor
77 RequestsInstrumentor = _NoOpRequestsInstrumentor
80class Swagger_Api(Api):
81 """
82 This is a modification of the base Flask Restplus Api class due to the issue described here
83 https://github.com/noirbizarre/flask-restplus/issues/223
84 """
86 @property
87 def specs_url(self):
88 return url_for(self.endpoint("specs"), _external=False)
91def get_last_compatible_gui_version() -> Version | bool:
92 logger.debug("Getting last compatible frontend...")
93 try:
94 res = requests.get(
95 "https://mindsdb-web-builds.s3.amazonaws.com/compatible-config.json",
96 timeout=5,
97 )
98 except (ConnectionError, requests.exceptions.ConnectionError) as e:
99 logger.error(f"Is no connection. {e}")
100 return False
101 except Exception as e:
102 logger.error(f"Is something wrong with getting compatible-config.json: {e}")
103 return False
105 if res.status_code != 200:
106 logger.error(f"Cant get compatible-config.json: returned status code = {res.status_code}")
107 return False
109 try:
110 versions = res.json()
111 except Exception as e:
112 logger.error(f"Cant decode compatible-config.json: {e}")
113 return False
115 current_mindsdb_lv = parse_version(mindsdb_version)
117 try:
118 gui_versions = {}
119 max_mindsdb_lv = None
120 max_gui_lv = None
121 for el in versions["mindsdb"]:
122 if el["mindsdb_version"] is None:
123 gui_lv = parse_version(el["gui_version"])
124 else:
125 mindsdb_lv = parse_version(el["mindsdb_version"])
126 gui_lv = parse_version(el["gui_version"])
127 if mindsdb_lv.base_version not in gui_versions or gui_lv > gui_versions[mindsdb_lv.base_version]:
128 gui_versions[mindsdb_lv.base_version] = gui_lv
129 if max_mindsdb_lv is None or max_mindsdb_lv < mindsdb_lv:
130 max_mindsdb_lv = mindsdb_lv
131 if max_gui_lv is None or max_gui_lv < gui_lv:
132 max_gui_lv = gui_lv
134 all_mindsdb_lv = [parse_version(x) for x in gui_versions.keys()]
135 all_mindsdb_lv.sort()
137 if current_mindsdb_lv.base_version in gui_versions:
138 gui_version_lv = gui_versions[current_mindsdb_lv.base_version]
139 elif current_mindsdb_lv > all_mindsdb_lv[-1]:
140 gui_version_lv = max_gui_lv
141 else:
142 lower_versions = {
143 key: value for key, value in gui_versions.items() if parse_version(key) < current_mindsdb_lv
144 }
145 if len(lower_versions) == 0:
146 gui_version_lv = gui_versions[all_mindsdb_lv[0].base_version]
147 else:
148 all_lower_versions = [parse_version(x) for x in lower_versions.keys()]
149 gui_version_lv = gui_versions[all_lower_versions[-1].base_version]
150 except Exception:
151 logger.exception("Error in compatible-config.json structure")
152 return False
154 logger.debug(f"Last compatible frontend version: {gui_version_lv}.")
155 return gui_version_lv
158def get_current_gui_version() -> Version:
159 logger.debug("Getting current frontend version...")
160 static_path = Path(config["paths"]["static"])
161 version_txt_path = static_path.joinpath("version.txt")
163 current_gui_version = None
164 if version_txt_path.is_file():
165 with open(version_txt_path, "rt") as f:
166 current_gui_version = f.readline()
168 current_gui_lv = None if current_gui_version is None else parse_version(current_gui_version)
169 logger.debug(f"Current frontend version: {current_gui_lv}.")
171 return current_gui_lv
174def initialize_static():
175 last_gui_version_lv = get_last_compatible_gui_version()
176 current_gui_version_lv = get_current_gui_version()
177 required_gui_version = config["gui"].get("version")
179 if required_gui_version is not None:
180 required_gui_version_lv = parse_version(required_gui_version)
181 success = True
182 if current_gui_version_lv is None or required_gui_version_lv != current_gui_version_lv:
183 success = update_static(required_gui_version_lv)
184 else:
185 if last_gui_version_lv is False:
186 logger.debug(
187 "The number of the latest version has not been determined, "
188 f"so we will continue using the current version: {current_gui_version_lv}"
189 )
190 return False
192 if current_gui_version_lv == last_gui_version_lv:
193 logger.debug(f"The latest version is already in use: {current_gui_version_lv}")
194 return True
195 success = update_static(last_gui_version_lv)
197 if db.session:
198 db.session.close()
199 return success
202def initialize_app(is_restart: bool = False):
203 static_root = config["paths"]["static"]
204 logger.debug(f"Static route: {static_root}")
205 init_static_thread = None
206 if not is_restart: 206 ↛ 217line 206 didn't jump to line 217 because the condition on line 206 was always true
207 gui_exists = Path(static_root).joinpath("index.html").is_file()
208 logger.debug(f"Does GUI already exist.. {'YES' if gui_exists else 'NO'}")
210 if config["gui"]["autoupdate"] is True or (config["gui"]["open_on_start"] is True and gui_exists is False): 210 ↛ 211line 210 didn't jump to line 211 because the condition on line 210 was never true
211 logger.debug("Initializing static...")
212 init_static_thread = threading.Thread(target=initialize_static, name="initialize_static")
213 init_static_thread.start()
214 else:
215 logger.debug(f"Skip initializing static: config['gui']={config['gui']}, gui_exists={gui_exists}")
217 app, api = initialize_flask()
219 if not is_restart and config["gui"]["open_on_start"]: 219 ↛ 220line 219 didn't jump to line 220 because the condition on line 219 was never true
220 if init_static_thread is not None:
221 init_static_thread.join()
222 open_gui(init_static_thread)
224 Compress(app)
226 initialize_interfaces(app)
228 if os.path.isabs(static_root) is False: 228 ↛ 229line 228 didn't jump to line 229 because the condition on line 228 was never true
229 static_root = os.path.join(os.getcwd(), static_root)
230 static_root = Path(static_root)
232 @app.route("/", defaults={"path": ""}, methods=["GET"])
233 @app.route("/<path:path>", methods=["GET"])
234 def root_index(path):
235 if path.startswith("api/"):
236 return http_error(
237 HTTPStatus.NOT_FOUND,
238 "Not found",
239 "The endpoint you are trying to access does not exist on the server.",
240 )
242 try:
243 # Ensure the requested path is within the static directory
244 # https://docs.python.org/3/library/pathlib.html#pathlib.PurePath.is_relative_to
245 requested_path = (static_root / path).resolve()
247 if not requested_path.is_relative_to(static_root.resolve()):
248 return http_error(
249 HTTPStatus.FORBIDDEN,
250 "Forbidden",
251 "You are not allowed to access the requested resource.",
252 )
254 if requested_path.is_file():
255 return send_from_directory(static_root, path)
256 else:
257 return send_from_directory(static_root, "index.html")
259 except (ValueError, OSError):
260 return http_error(
261 HTTPStatus.BAD_REQUEST,
262 "Bad Request",
263 "Invalid path requested.",
264 )
266 protected_namespaces = [
267 tab_ns,
268 utils_ns,
269 conf_ns,
270 file_ns,
271 sql_ns,
272 analysis_ns,
273 handlers_ns,
274 tree_ns,
275 projects_ns,
276 databases_ns,
277 views_ns,
278 models_ns,
279 chatbots_ns,
280 skills_ns,
281 agents_ns,
282 jobs_ns,
283 knowledge_bases_ns,
284 ]
286 for ns in protected_namespaces:
287 api.add_namespace(ns)
288 api.add_namespace(default_ns)
289 api.add_namespace(auth_ns)
290 api.add_namespace(webhooks_ns)
292 @api.errorhandler(Exception)
293 def handle_exception(e):
294 logger.error(f"http exception: {e}")
295 # pass through HTTP errors
296 # NOTE flask_restx require 'message', also it modyfies 'application/problem+json' to 'application/json'
297 if isinstance(e, HTTPException): 297 ↛ 298line 297 didn't jump to line 298 because the condition on line 297 was never true
298 return (
299 {"title": e.name, "detail": e.description, "message": e.description},
300 e.code,
301 {"Content-Type": "application/problem+json"},
302 )
303 return (
304 {
305 "title": getattr(type(e), "__name__") or "Unknown error",
306 "detail": str(e),
307 "message": str(e),
308 },
309 500,
310 {"Content-Type": "application/problem+json"},
311 )
313 @app.teardown_appcontext
314 def remove_session(*args, **kwargs):
315 db.session.remove()
317 @app.before_request
318 def before_request():
319 ctx.set_default()
321 h = request.headers.get("Authorization")
322 if not h or not h.startswith("Bearer "): 322 ↛ 325line 322 didn't jump to line 325 because the condition on line 322 was always true
323 bearer = None
324 else:
325 bearer = h.split(" ", 1)[1].strip() or None
327 # region routes where auth is required
328 http_auth_type = config["auth"]["http_auth_type"]
329 if ( 329 ↛ 342line 329 didn't jump to line 342 because the condition on line 329 was never true
330 config["auth"]["http_auth_enabled"] is True
331 and any(request.path.startswith(f"/api{ns.path}") for ns in protected_namespaces)
332 and (
333 (http_auth_type == HTTP_AUTH_TYPE.SESSION and check_session_auth() is False)
334 or (http_auth_type == HTTP_AUTH_TYPE.TOKEN and verify_pat(bearer) is False)
335 or (
336 http_auth_type == HTTP_AUTH_TYPE.SESSION_OR_TOKEN
337 and check_session_auth() is False
338 and verify_pat(bearer) is False
339 )
340 )
341 ):
342 logger.debug(f"Auth failed for path {request.path}")
343 return http_error(
344 HTTPStatus.UNAUTHORIZED,
345 "Unauthorized",
346 "Authorization is required to complete the request",
347 )
348 # endregion
350 company_id = request.headers.get("company-id")
351 user_class = request.headers.get("user-class")
353 try:
354 email_confirmed = int(request.headers.get("email-confirmed", 1))
355 except Exception:
356 email_confirmed = 1
358 try:
359 user_id = int(request.headers.get("user-id", 0))
360 except Exception:
361 user_id = 0
363 if user_class is not None: 363 ↛ 364line 363 didn't jump to line 364 because the condition on line 363 was never true
364 try:
365 user_class = int(user_class)
366 except Exception as e:
367 logger.error(f"Could not parse user_class: {user_class} | exception: {e}")
368 user_class = 0
369 else:
370 user_class = 0
372 ctx.user_id = user_id
373 ctx.company_id = company_id
374 ctx.user_class = user_class
375 ctx.email_confirmed = email_confirmed
377 logger.debug("Done initializing app.")
378 return app
381def initialize_flask():
382 logger.debug("Initializing flask...")
383 # region required for windows https://github.com/mindsdb/mindsdb/issues/2526
384 mimetypes.add_type("text/css", ".css")
385 mimetypes.add_type("text/javascript", ".js")
386 # endregion
388 static_path = os.path.join(config["paths"]["static"], "static/")
389 if os.path.isabs(static_path) is False: 389 ↛ 390line 389 didn't jump to line 390 because the condition on line 389 was never true
390 static_path = os.path.join(os.getcwd(), static_path)
391 kwargs = {"static_url_path": "/static", "static_folder": static_path}
392 logger.debug(f"Static path: {static_path}")
394 app = Flask(__name__, **kwargs)
395 init_metrics(app)
397 # Instrument Flask app and requests using either real or no-op instrumentors
398 FlaskInstrumentor().instrument_app(app)
399 RequestsInstrumentor().instrument()
401 app.config["SEND_FILE_MAX_AGE_DEFAULT"] = 60
402 app.config["SWAGGER_HOST"] = "http://localhost:8000/mindsdb"
403 app.json = ORJSONProvider(app)
405 http_auth_type = config["auth"]["http_auth_type"]
406 authorizations = {}
407 security = []
409 if http_auth_type in (HTTP_AUTH_TYPE.SESSION, HTTP_AUTH_TYPE.SESSION_OR_TOKEN): 409 ↛ 416line 409 didn't jump to line 416 because the condition on line 409 was always true
410 app.config["SECRET_KEY"] = os.environ.get("FLASK_SECRET_KEY", secrets.token_hex(32))
411 app.config["SESSION_COOKIE_NAME"] = "session"
412 app.config["PERMANENT_SESSION_LIFETIME"] = config["auth"]["http_permanent_session_lifetime"]
413 authorizations["session"] = {"type": "apiKey", "in": "cookie", "name": "session"}
414 security.append(["session"])
416 if http_auth_type in (HTTP_AUTH_TYPE.TOKEN, HTTP_AUTH_TYPE.SESSION_OR_TOKEN): 416 ↛ 420line 416 didn't jump to line 420 because the condition on line 416 was always true
417 authorizations["bearer"] = {"type": "apiKey", "in": "header", "name": "Authorization"}
418 security.append(["bearer"])
420 logger.debug("Creating swagger API..")
421 api = Swagger_Api(
422 app,
423 authorizations=authorizations,
424 security=security,
425 url_prefix=":8000",
426 prefix="/api",
427 doc="/doc/",
428 )
430 def __output_json_orjson(data, code, headers=None):
431 from flask import current_app, make_response
433 dumped = current_app.json.dumps(data)
434 resp = make_response(dumped, code)
435 if headers:
436 resp.headers.extend(headers)
437 resp.mimetype = "application/json"
438 return resp
440 api.representations["application/json"] = __output_json_orjson
442 return app, api
445def open_gui(init_static_thread):
446 port = config["api"]["http"]["port"]
447 host = config["api"]["http"]["host"]
449 if host in ("", "0.0.0.0"):
450 url = f"http://127.0.0.1:{port}/"
451 else:
452 url = f"http://{host}:{port}/"
453 logger.info(f" - GUI available at {url}")
455 pid = os.getpid()
456 thread = threading.Thread(
457 target=_open_webbrowser,
458 args=(url, pid, port, init_static_thread, config["paths"]["static"]),
459 daemon=True,
460 name="open_webbrowser",
461 )
462 thread.start()
465def initialize_interfaces(app):
466 app.integration_controller = integration_controller
467 app.database_controller = DatabaseController()
468 app.file_controller = FileController()
469 app.jobs_controller = JobsController()
472def _open_webbrowser(url: str, pid: int, port: int, init_static_thread, static_folder):
473 """Open webbrowser with url when http service is started.
475 If some error then do nothing.
476 """
477 if init_static_thread is not None:
478 init_static_thread.join()
479 try:
480 is_http_active = wait_func_is_true(func=is_pid_listen_port, timeout=15, pid=pid, port=port)
481 if is_http_active:
482 webbrowser.open(url)
483 except Exception:
484 logger.exception(f"Failed to open {url} in webbrowser with exception:")
485 db.session.close()