Coverage for mindsdb / utilities / otel / prepare.py: 0%

1import os 

2import typing 

3 

4from opentelemetry import trace # noqa: F401 

5from opentelemetry import metrics # noqa: F401 

6from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter as OTLPLogExporterGRPC 

7from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter as OTLPLogExporterHTTP 

8from opentelemetry.sdk._logs._internal.export import LogExporter 

9from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter as OTLPMetricExporterGRPC 

10from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter as OTLPMetricExporterHTTP 

11from opentelemetry.sdk.metrics.export import MetricExporter, ConsoleMetricExporter 

12from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter as OTLPSpanExporterGRPC 

13from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter as OTLPSpanExporterHTTP 

14from opentelemetry.sdk.trace.export import SpanExporter, ConsoleSpanExporter 

15from opentelemetry.sdk.resources import Resource 

16from opentelemetry.sdk.trace.sampling import TraceIdRatioBased 

17 

18from mindsdb.utilities.otel.logger import setup_logger 

19from mindsdb.utilities.otel.meter import setup_meter 

20from mindsdb.utilities.otel.tracer import setup_tracer 

21from mindsdb.utilities.utils import parse_csv_attributes 

22from mindsdb.utilities import log 

23 

24logger = log.getLogger(__name__) 

25 

26# Check OpenTelemetry exporter type 

27OTEL_EXPORTER_TYPE = os.getenv("OTEL_EXPORTER_TYPE", "console") # console or otlp 

28 

29# Define OpenTelemetry exporter protocol 

30OTEL_EXPORTER_PROTOCOL = os.getenv("OTEL_EXPORTER_PROTOCOL", "grpc") # grpc or http 

31 

32# Define OTLP endpoint. If not set, the default OTLP endpoint will be used 

33OTEL_OTLP_ENDPOINT = os.getenv("OTEL_OTLP_ENDPOINT", "http://localhost:4317") 

34 

35# Define OTLP logging endpoint. If not set, the default OTLP logging endpoint will be used 

36OTEL_OTLP_LOGGING_ENDPOINT = os.getenv("OTEL_OTLP_LOGGING_ENDPOINT", OTEL_OTLP_ENDPOINT) 

37 

38# Define OTLP tracing endpoint. If not set, the default OTLP tracing endpoint will be used 

39OTEL_OTLP_TRACING_ENDPOINT = os.getenv("OTEL_OTLP_TRACING_ENDPOINT", OTEL_OTLP_ENDPOINT) 

40 

41# Define OTLP metrics endpoint. If not set, the default OTLP metrics endpoint will be used 

42OTEL_OTLP_METRICS_ENDPOINT = os.getenv("OTEL_OTLP_METRICS_ENDPOINT", OTEL_OTLP_ENDPOINT) 

43 

44# Define service name 

45OTEL_SERVICE_NAME = os.getenv("OTEL_SERVICE_NAME", "mindsdb") 

46 

47# Define service instace ID 

48OTEL_SERVICE_INSTANCE_ID = os.getenv("OTEL_SERVICE_INSTANCE_ID", "mindsdb-instance") 

49 

50# The name of the environment we"re on, by default local for development, this is set differently per-env in our Helm 

51# chart values files 

52OTEL_SERVICE_ENVIRONMENT = os.getenv("OTEL_SERVICE_ENVIRONMENT", "local").lower() 

53 

54# Define service release 

55OTEL_SERVICE_RELEASE = os.getenv("OTEL_SERVICE_RELEASE", "local").lower() 

56 

57# Define how often to capture traces 

58OTEL_TRACE_SAMPLE_RATE = float(os.getenv("OTEL_TRACE_SAMPLE_RATE", "1.0")) 

59 

60# Define extra attributes 

61OTEL_EXTRA_ATTRIBUTES = os.getenv("OTEL_EXTRA_ATTRIBUTES", "") 

62 

63# Define if OpenTelemetry logging is disabled. By default, it is disabled. 

64OTEL_LOGGING_DISABLED = os.getenv("OTEL_LOGGING_DISABLED", "true").lower() == "true" 

65 

66# Define if OpenTelemetry tracing is disabled. By default, it is enabled. 

67OTEL_TRACING_DISABLED = os.getenv("OTEL_TRACING_DISABLED", "false").lower() == "true" 

68 

69# Define if OpenTelemetry metrics is disabled. By default, it is disabled. 

70OTEL_METRICS_DISABLED = os.getenv("OTEL_METRICS_DISABLED", "true").lower() == "true" 

71 

72 

73def get_otel_attributes() -> dict: 

74 """ 

75 Get OpenTelemetry attributes 

76 

77 Returns: 

78 dict: OpenTelemetry attributes 

79 """ 

80 

81 base_attributes = { 

82 "service.name": OTEL_SERVICE_NAME, 

83 "service.instance.id": OTEL_SERVICE_INSTANCE_ID, 

84 "environment": OTEL_SERVICE_ENVIRONMENT, 

85 "release": OTEL_SERVICE_RELEASE, 

86 } 

87 

88 extra_attributes = {} 

89 try: 

90 extra_attributes = parse_csv_attributes(OTEL_EXTRA_ATTRIBUTES) 

91 except Exception as e: 

92 logger.error(f"Failed to parse OTEL_EXTRA_ATTRIBUTES: {e}") 

93 

94 attributes = {**extra_attributes, **base_attributes} # Base attributes take precedence over extra attributes 

95 

96 return attributes 

97 

98 

99def get_logging_exporter() -> typing.Optional[LogExporter]: 

100 """ 

101 Get OpenTelemetry logging exporter. 

102 

103 Returns: 

104 OTLPLogExporter: OpenTelemetry logging exporter 

105 """ 

106 

107 if OTEL_EXPORTER_TYPE == "otlp": 

108 

109 if OTEL_EXPORTER_PROTOCOL == "grpc": 

110 return OTLPLogExporterGRPC( 

111 endpoint=OTEL_OTLP_LOGGING_ENDPOINT, 

112 insecure=True 

113 ) 

114 

115 elif OTEL_EXPORTER_PROTOCOL == "http": 

116 return OTLPLogExporterHTTP( 

117 endpoint=OTEL_OTLP_LOGGING_ENDPOINT 

118 ) 

119 

120 return None 

121 

122 

123def get_span_exporter() -> SpanExporter: 

124 """ 

125 Get OpenTelemetry span exporter 

126 

127 Returns: 

128 OTLPSpanExporter: OpenTelemetry span exporter 

129 """ 

130 

131 if OTEL_EXPORTER_TYPE == "otlp": 

132 

133 if OTEL_EXPORTER_PROTOCOL == "grpc": 

134 return OTLPSpanExporterGRPC( 

135 endpoint=OTEL_OTLP_TRACING_ENDPOINT, 

136 insecure=True 

137 ) 

138 

139 elif OTEL_EXPORTER_PROTOCOL == "http": 

140 return OTLPSpanExporterHTTP( 

141 endpoint=OTEL_OTLP_TRACING_ENDPOINT 

142 ) 

143 

144 return ConsoleSpanExporter() 

145 

146 

147def get_metrics_exporter() -> typing.Optional[MetricExporter]: 

148 """ 

149 Get OpenTelemetry metrics exporter 

150 

151 Returns: 

152 OTLPLogExporter: OpenTelemetry metrics exporter 

153 """ 

154 

155 if OTEL_EXPORTER_TYPE == "otlp": 

156 

157 if OTEL_EXPORTER_PROTOCOL == "grpc": 

158 return OTLPMetricExporterGRPC( 

159 endpoint=OTEL_OTLP_METRICS_ENDPOINT, 

160 insecure=True 

161 ) 

162 

163 elif OTEL_EXPORTER_PROTOCOL == "http": 

164 return OTLPMetricExporterHTTP( 

165 endpoint=OTEL_OTLP_METRICS_ENDPOINT 

166 ) 

167 

168 return ConsoleMetricExporter() 

169 

170 

171logger.info("OpenTelemetry enabled") 

172logger.info(f"OpenTelemetry exporter type: {OTEL_EXPORTER_TYPE}") 

173logger.info(f"OpenTelemetry service name: {OTEL_SERVICE_NAME}") 

174logger.info(f"OpenTelemetry service environment: {OTEL_SERVICE_ENVIRONMENT}") 

175logger.info(f"OpenTelemetry service release: {OTEL_SERVICE_RELEASE}") 

176logger.info(f"OpenTelemetry trace sample rate: {OTEL_TRACE_SAMPLE_RATE}") 

177logger.info(f"OpenTelemetry extra attributes: {OTEL_EXTRA_ATTRIBUTES}") 

178 

179# Define OpenTelemetry resources (e.g., service name) 

180attributes = get_otel_attributes() 

181 

182# Define OpenTelemetry sampler 

183sampler = TraceIdRatioBased(OTEL_TRACE_SAMPLE_RATE) 

184 

185# Define OpenTelemetry resources (e.g., service name) 

186resource = Resource(attributes=attributes) 

187 

188if not OTEL_LOGGING_DISABLED: 

189 logger.info("OpenTelemetry Logging is enabled") 

190 setup_logger(resource, get_logging_exporter()) 

191 

192if not OTEL_TRACING_DISABLED: 

193 logger.info("OpenTelemetry Tracing is enabled") 

194 setup_tracer(resource, sampler, get_span_exporter()) 

195 

196if not OTEL_METRICS_DISABLED: 

197 logger.info("OpenTelemetry Metrics is enabled") 

198 setup_meter(resource, get_metrics_exporter())