Coverage for mindsdb / integrations / handlers / gitlab_handler / gitlab_tables.py: 0%

1import pandas as pd 

2 

3from typing import List 

4 

5from mindsdb.integrations.libs.api_handler import APITable 

6from mindsdb.integrations.utilities.sql_utils import extract_comparison_conditions 

7from mindsdb.utilities import log 

8 

9from mindsdb_sql_parser import ast 

10 

11logger = log.getLogger(__name__) 

12 

13 

14class GitlabIssuesTable(APITable): 

15 """The GitLab Issue Table implementation""" 

16 

17 def select(self, query: ast.Select) -> pd.DataFrame: 

18 """Pulls data from the GitLab "List repository issues" API 

19 Args: 

20 query: SELECT 

21 Returns: 

22 DataFrame 

23 Raises: 

24 ValueError 

25 """ 

26 

27 conditions = extract_comparison_conditions(query.where) 

28 

29 if query.limit: 

30 total_results = query.limit.value 

31 else: 

32 total_results = 20 

33 

34 issues_kwargs = {} 

35 order_by_conditions = {} 

36 

37 if query.order_by and len(query.order_by) > 0: 

38 order_by_conditions["columns"] = [] 

39 order_by_conditions["ascending"] = [] 

40 

41 for an_order in query.order_by: 

42 if an_order.field.parts[1] in self.get_columns(): 

43 order_by_conditions["columns"].append(an_order.field.parts[1]) 

44 

45 if an_order.direction == "ASC": 

46 order_by_conditions["ascending"].append(True) 

47 else: 

48 order_by_conditions["ascending"].append(False) 

49 else: 

50 raise ValueError( 

51 f"Order by unknown column {an_order.field.parts[1]}" 

52 ) 

53 

54 for a_where in conditions: 

55 if a_where[1] == "state": 

56 if a_where[0] != "=": 

57 raise ValueError("Unsupported where operation for state") 

58 if a_where[2] not in ["opened", "closed", "all"]: 

59 raise ValueError( 

60 f"Unsupported where argument for state {a_where[2]}" 

61 ) 

62 

63 issues_kwargs["state"] = a_where[2] 

64 

65 continue 

66 if a_where[1] == "labels": 

67 if a_where[0] != "=": 

68 raise ValueError("Unsupported where operation for state") 

69 

70 issues_kwargs["labels"] = a_where[2].split(",") 

71 

72 continue 

73 if a_where[1] in ["assignee", "creator"]: 

74 if a_where[0] != "=": 

75 raise ValueError(f"Unsupported where operation for {a_where[1]}") 

76 

77 issues_kwargs[a_where[1]] = a_where[2] 

78 else: 

79 raise ValueError(f"Unsupported where argument {a_where[1]}") 

80 

81 self.handler.connect() 

82 

83 gitlab_issues_df = pd.DataFrame(columns=self.get_columns()) 

84 

85 issues_kwargs["per_page"] = total_results 

86 issues_kwargs["get_all"] = False 

87 while True: 

88 try: 

89 for issue in self.handler.connection.projects.get( 

90 self.handler.repository 

91 ).issues.list(**issues_kwargs): 

92 

93 logger.debug(f"Processing issue {issue.iid}") 

94 

95 gitlab_issues_df = pd.concat( 

96 [ 

97 gitlab_issues_df, 

98 pd.DataFrame( 

99 [ 

100 { 

101 "number": issue.iid, 

102 "title": issue.title, 

103 "state": issue.state, 

104 "creator": issue.author["name"], 

105 "closed_by": issue.closed_by 

106 if issue.closed_by 

107 else None, 

108 "labels": ",".join( 

109 [label for label in issue.labels] 

110 ), 

111 "assignees": ",".join( 

112 [ 

113 assignee["name"] 

114 for assignee in issue.assignees 

115 ] 

116 ), 

117 "body": issue.description, 

118 "created": issue.created_at, 

119 "updated": issue.updated_at, 

120 "closed": issue.closed_at, 

121 } 

122 ] 

123 ), 

124 ] 

125 ) 

126 

127 if gitlab_issues_df.shape[0] >= total_results: 

128 break 

129 except IndexError: 

130 break 

131 

132 if gitlab_issues_df.shape[0] >= total_results: 

133 break 

134 else: 

135 break 

136 

137 selected_columns = [] 

138 for target in query.targets: 

139 if isinstance(target, ast.Star): 

140 selected_columns = self.get_columns() 

141 break 

142 elif isinstance(target, ast.Identifier): 

143 selected_columns.append(target.parts[-1]) 

144 else: 

145 raise ValueError(f"Unknown query target {type(target)}") 

146 

147 if len(gitlab_issues_df) == 0: 

148 gitlab_issues_df = pd.DataFrame([], columns=selected_columns) 

149 else: 

150 gitlab_issues_df.columns = self.get_columns() 

151 for col in set(gitlab_issues_df.columns).difference(set(selected_columns)): 

152 gitlab_issues_df = gitlab_issues_df.drop(col, axis=1) 

153 

154 if len(order_by_conditions.get("columns", [])) > 0: 

155 gitlab_issues_df = gitlab_issues_df.sort_values( 

156 by=order_by_conditions["columns"], 

157 ascending=order_by_conditions["ascending"], 

158 ) 

159 

160 return gitlab_issues_df 

161 

162 def get_columns(self) -> List[str]: 

163 """Gets all columns to be returned in pandas DataFrame responses 

164 Returns 

165 ------- 

166 List[str]: list of columns 

167 """ 

168 

169 return [ 

170 "number", 

171 "title", 

172 "state", 

173 "creator", 

174 "closed_by", 

175 "labels", 

176 "assignees", 

177 "body", 

178 "created", 

179 "updated", 

180 "closed", 

181 ] 

182 

183 

184class GitlabMergeRequestsTable(APITable): 

185 """The GitLab Merge Requests Table implementation""" 

186 

187 def select(self, query: ast.Select) -> pd.DataFrame: 

188 """Pulls data from the GitLab "List repository rerge requests" API 

189 Args: 

190 query: SELECT 

191 Returns: 

192 DataFrame 

193 Raises: 

194 ValueError 

195 """ 

196 

197 conditions = extract_comparison_conditions(query.where) 

198 

199 if query.limit: 

200 total_results = query.limit.value 

201 else: 

202 total_results = 20 

203 

204 merge_requests_kwargs = {} 

205 order_by_conditions = {} 

206 

207 if query.order_by and len(query.order_by) > 0: 

208 order_by_conditions["columns"] = [] 

209 order_by_conditions["ascending"] = [] 

210 

211 for an_order in query.order_by: 

212 if an_order.field.parts[1] in self.get_columns(): 

213 order_by_conditions["columns"].append(an_order.field.parts[1]) 

214 

215 if an_order.direction == "ASC": 

216 order_by_conditions["ascending"].append(True) 

217 else: 

218 order_by_conditions["ascending"].append(False) 

219 else: 

220 raise ValueError( 

221 f"Order by unknown column {an_order.field.parts[1]}" 

222 ) 

223 

224 for a_where in conditions: 

225 if a_where[1] == "state": 

226 if a_where[0] != "=": 

227 raise ValueError("Unsupported where operation for state") 

228 if a_where[2] not in ["opened", "closed", "merged", "all"]: 

229 raise ValueError( 

230 f"Unsupported where argument for state {a_where[2]}" 

231 ) 

232 

233 merge_requests_kwargs["state"] = a_where[2] 

234 

235 continue 

236 if a_where[1] == "labels": 

237 if a_where[0] != "=": 

238 raise ValueError("Unsupported where operation for labels") 

239 

240 merge_requests_kwargs["labels"] = a_where[2].split(",") 

241 

242 continue 

243 if a_where[1] in ["target_branch", "source_branch"]: 

244 if a_where[0] != "=": 

245 raise ValueError(f"Unsupported where operation for {a_where[1]}") 

246 

247 merge_requests_kwargs[a_where[1]] = a_where[2] 

248 else: 

249 raise ValueError(f"Unsupported where argument {a_where[1]}") 

250 

251 self.handler.connect() 

252 

253 gitlab_merge_requests_df = pd.DataFrame(columns=self.get_columns()) 

254 

255 merge_requests_kwargs["per_page"] = total_results 

256 merge_requests_kwargs["get_all"] = False 

257 while True: 

258 try: 

259 for merge_request in self.handler.connection.projects.get( 

260 self.handler.repository 

261 ).mergerequests.list(**merge_requests_kwargs): 

262 

263 logger.debug(f"Processing merge request {merge_request.iid}") 

264 

265 gitlab_merge_requests_df = pd.concat( 

266 [ 

267 gitlab_merge_requests_df, 

268 pd.DataFrame( 

269 [ 

270 { 

271 "number": merge_request.iid, 

272 "title": merge_request.title, 

273 "state": merge_request.state, 

274 "creator": merge_request.author["name"], 

275 "closed_by": merge_request.closed_by 

276 if merge_request.closed_by 

277 else None, 

278 "mergeed_by": merge_request.merge_user["name"] 

279 if merge_request.merge_user 

280 else None, 

281 "labels": ",".join( 

282 [label for label in merge_request.labels] 

283 ), 

284 "assignees": ",".join( 

285 [ 

286 assignee["name"] 

287 for assignee in merge_request.assignees 

288 ] 

289 ), 

290 "reviewers": ",".join( 

291 [ 

292 reviewer["name"] 

293 for reviewer in merge_request.reviewers 

294 ] 

295 ), 

296 "body": merge_request.description, 

297 "target_branch": merge_request.target_branch, 

298 "source_branch": merge_request.source_branch, 

299 "upvotes": merge_request.upvotes, 

300 "downvotes": merge_request.downvotes, 

301 "draft": merge_request.draft, 

302 "work_in_progress": merge_request.work_in_progress, 

303 "milestone": merge_request.milestone["state"] 

304 if merge_request.milestone 

305 else None, 

306 "merge_status": merge_request.merge_status, 

307 "detailed_merge_status": merge_request.detailed_merge_status, 

308 "user_notes_count": merge_request.user_notes_count, 

309 "has_conflicts": merge_request.has_conflicts, 

310 "blocking_discussions_resolved": merge_request.blocking_discussions_resolved, 

311 "created": merge_request.created_at, 

312 "updated": merge_request.updated_at, 

313 "closed": merge_request.closed_at, 

314 "merged": merge_request.merged_at, 

315 } 

316 ] 

317 ), 

318 ] 

319 ) 

320 

321 if gitlab_merge_requests_df.shape[0] >= total_results: 

322 break 

323 except IndexError: 

324 break 

325 

326 if gitlab_merge_requests_df.shape[0] >= total_results: 

327 break 

328 else: 

329 break 

330 

331 selected_columns = [] 

332 for target in query.targets: 

333 if isinstance(target, ast.Star): 

334 selected_columns = self.get_columns() 

335 break 

336 elif isinstance(target, ast.Identifier): 

337 selected_columns.append(target.parts[-1]) 

338 else: 

339 raise ValueError(f"Unknown query target {type(target)}") 

340 

341 if len(gitlab_merge_requests_df) == 0: 

342 gitlab_merge_requests_df = pd.DataFrame([], columns=selected_columns) 

343 else: 

344 gitlab_merge_requests_df.columns = self.get_columns() 

345 for col in set(gitlab_merge_requests_df.columns).difference(set(selected_columns)): 

346 gitlab_merge_requests_df = gitlab_merge_requests_df.drop(col, axis=1) 

347 

348 if len(order_by_conditions.get("columns", [])) > 0: 

349 gitlab_merge_requests_df = gitlab_merge_requests_df.sort_values( 

350 by=order_by_conditions["columns"], 

351 ascending=order_by_conditions["ascending"], 

352 ) 

353 

354 return gitlab_merge_requests_df 

355 

356 def get_columns(self) -> List[str]: 

357 """Gets all columns to be returned in pandas DataFrame responses 

358 Returns 

359 ------- 

360 List[str]: list of columns 

361 """ 

362 

363 return [ 

364 "number", 

365 "title", 

366 "state", 

367 "creator", 

368 "closed_by", 

369 "mergeed_by", 

370 "labels", 

371 "assignees", 

372 "reviewers", 

373 "body", 

374 "target_branch", 

375 "source_branch", 

376 "upvotes", 

377 "downvotes", 

378 "draft", 

379 "work_in_progress", 

380 "milestone", 

381 "merge_status", 

382 "detailed_merge_status", 

383 "user_notes_count", 

384 "has_conflicts", 

385 "blocking_discussions_resolved", 

386 "created", 

387 "updated", 

388 "closed", 

389 "merged", 

390 ]