Coverage for mindsdb / integrations / handlers / github_handler / generate_api.py: 0%

1import inspect 

2from typing import List 

3from dataclasses import dataclass 

4 

5import pandas as pd 

6import github 

7 

8from mindsdb.integrations.utilities.sql_utils import (FilterCondition, FilterOperator, SortColumn) 

9from mindsdb.integrations.libs.api_handler import APIResource 

10 

11 

12@dataclass 

13class Type: 

14 name: str 

15 sub_type: str = None 

16 optional: bool = False 

17 

18 

19@dataclass 

20class GHMethod: 

21 name: str 

22 table_name: str 

23 params: dict 

24 output: Type 

25 

26 

27def parse_annotations(annotations): 

28 ''' 

29 Parse string annotation, and extract type, input examples: 

30 - Milestone | Opt[str] 

31 - PaginatedList[Issue] 

32 ''' 

33 type_name, sub_type = None, None 

34 if not isinstance(annotations, str): 

35 

36 return Type(getattr(annotations, '__name__', None)) 

37 for item in annotations.split('|'): 

38 item = item.strip() 

39 if item is None: 

40 continue 

41 if '[' in item: 

42 type_name = item[: item.find('[')] 

43 item2 = item[item.find('[') + 1: item.rfind(']')] 

44 if type_name == 'Opt': 

45 inner_type = parse_annotations(item2) 

46 inner_type.optional = Type 

47 return inner_type 

48 if type_name == 'dict': 

49 item2 = item2[item2.find(',') + 1:] 

50 sub_type = parse_annotations(item2).name 

51 else: 

52 type_name = item 

53 # get only first type 

54 break 

55 return Type(type_name, sub_type) 

56 

57 

58def get_properties(cls): 

59 # find properties of the class 

60 

61 properties = {} 

62 for prop_name, prop in inspect.getmembers(cls): 

63 if prop_name.startswith('_'): 

64 continue 

65 if not isinstance(prop, property): 

66 continue 

67 sig2 = inspect.signature(prop.fget) 

68 

69 properties[prop_name] = parse_annotations(sig2.return_annotation) 

70 return properties 

71 

72 

73def get_github_types(): 

74 # get github types 

75 types = {} 

76 

77 GithubObject = github.GithubObject.GithubObject 

78 for module_name, module in inspect.getmembers(github, inspect.ismodule): 

79 cls = getattr(module, module_name, None) 

80 if cls is None: 

81 continue 

82 if issubclass(cls, GithubObject): 

83 

84 # remove inherited props 

85 parent_props = [] 

86 for cls2 in cls.__bases__: 

87 parent_props += get_properties(cls2).keys() 

88 

89 properties = {} 

90 for k, v in get_properties(cls).items(): 

91 if k not in parent_props: 

92 properties[k] = v 

93 

94 types[module_name] = properties 

95 return types 

96 

97 

98def get_github_methods(cls): 

99 ''' 

100 Analyse class in order to find methods which return list of objects. 

101 ''' 

102 methods = [] 

103 

104 for method_name, method in inspect.getmembers(cls, inspect.isfunction): 

105 sig = inspect.signature(method) 

106 

107 return_type = parse_annotations(sig.return_annotation) 

108 list_prefix = 'get_' 

109 if not (method_name.startswith(list_prefix) and return_type.name == 'PaginatedList'): 

110 continue 

111 

112 table_name = method_name[len(list_prefix):] 

113 

114 params = {} 

115 for param_name, param in sig.parameters.items(): 

116 params[param_name] = parse_annotations(param.annotation) 

117 

118 methods.append(GHMethod( 

119 name=method_name, 

120 table_name=table_name, 

121 params=params, 

122 output=return_type 

123 )) 

124 return methods 

125 

126 

127class GHTable(APIResource): 

128 def __init__(self, *args, method: GHMethod = None, github_types=None, **kwargs): 

129 self.method = method 

130 self.github_types = github_types 

131 

132 self.output_columns = {} 

133 if method.output.sub_type in self.github_types: 

134 self.output_columns = self.github_types[method.output.sub_type] 

135 

136 # check params: 

137 self.params, self.list_params = [], [] 

138 for name, param_type in method.params.items(): 

139 self.params.append(name) 

140 if param_type.name == 'list': 

141 self.list_params.append(name) 

142 

143 self._allow_sort = 'sort' in method.params 

144 

145 super().__init__(*args, **kwargs) 

146 

147 def repr_value(self, value, type_name): 

148 if value is None or type_name in ('bool', 'int', 'float'): 

149 return value 

150 if type_name in self.github_types: 

151 properties = self.github_types[type_name] 

152 if 'login' in properties: 

153 value = getattr(value, 'login') 

154 elif 'url' in properties: 

155 value = getattr(value, 'url') 

156 return str(value) 

157 

158 def get_columns(self) -> List[str]: 

159 return list(self.output_columns.keys()) 

160 

161 def list( 

162 self, 

163 conditions: List[FilterCondition] = None, 

164 limit: int = None, 

165 sort: List[SortColumn] = None, 

166 targets: List[str] = None, 

167 **kwargs 

168 ) -> pd.DataFrame: 

169 

170 if limit is None: 

171 limit = 20 

172 

173 method_kwargs = {} 

174 if sort is not None and self._allow_sort: 

175 for col in sort: 

176 method_kwargs['sort'] = col.column 

177 method_kwargs['direction'] = 'asc' if col.ascending else 'desc' 

178 sort.applied = True 

179 # supported only 1 column 

180 break 

181 

182 if conditions: 

183 for condition in conditions: 

184 if condition.column not in self.params: 

185 continue 

186 

187 if condition.column in self.list_params: 

188 if condition.op == FilterOperator.IN: 

189 method_kwargs[condition.column] = condition.value 

190 elif condition.op == FilterOperator.EQUAL: 

191 method_kwargs[condition.column] = [condition] 

192 condition.applied = True 

193 else: 

194 method_kwargs[condition.column] = condition.value 

195 condition.applied = True 

196 

197 connection = self.handler.connect() 

198 method = getattr(connection.get_repo(self.handler.repository), self.method.name) 

199 

200 data = [] 

201 count = 0 

202 for record in method(**method_kwargs): 

203 item = {} 

204 for name, output_type in self.output_columns.items(): 

205 

206 # workaround to prevent making addition request per property. 

207 if name in targets: 

208 # request only if is required 

209 value = getattr(record, name) 

210 else: 

211 value = getattr(record, '_' + name).value 

212 if value is not None: 

213 if output_type.name == 'list': 

214 value = ",".join([ 

215 str(self.repr_value(i, output_type.sub_type)) 

216 for i in value 

217 ]) 

218 else: 

219 value = self.repr_value(value, output_type.name) 

220 item[name] = value 

221 

222 data.append(item) 

223 

224 count += 1 

225 if limit <= count: 

226 break 

227 

228 return pd.DataFrame(data, columns=self.get_columns())