1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
open Ast
let registry = Hashtbl.create 16
let register name serializer =
Hashtbl.replace registry name serializer
let lookup name =
Hashtbl.find_opt registry name
let get_builtins () =
Hashtbl.to_seq registry |> List.of_seq
let update_native name ?writer ?reader () =
match lookup name with
| Some ser ->
let ser = { ser with
s_writer = (match writer with Some w -> w | None -> ser.s_writer);
s_reader = (match reader with Some r -> r | None -> ser.s_reader) }
in
register name ser
| None -> ()
let init_builtins () =
let not_implemented_builtin name =
VBuiltin {
b_name = Some ("^" ^ name ^ "_io");
b_arity = 2;
b_variadic = false;
b_func = (fun _ _ ->
Error.make_error RuntimeError (Printf.sprintf "Serializer ^%s does not have a T-native implementation yet. Use it in a pipeline with R or Python runtimes." name)
)
}
in
let mk_ser format =
let (r_w, r_r, py_w, py_r) =
match format with
| "csv" -> (Some "r_write_csv", Some "r_read_csv", Some "py_write_csv", Some "py_read_csv")
| "arrow" -> (Some "r_write_arrow", Some "r_read_arrow", Some "py_write_arrow", Some "py_read_arrow")
| "json" -> (Some "r_write_json", Some "r_read_json", Some "py_write_json", Some "py_read_json")
| "pmml" -> (Some "r_write_pmml", Some "r_read_pmml", Some "py_write_pmml", Some "py_read_pmml")
| "onnx" -> (Some "r_write_onnx", Some "r_read_onnx", Some "py_write_onnx", Some "py_read_onnx")
| "text" -> (Some "writeLines", Some "readLines", Some "lambda obj, path: open(path, 'w').write(str(obj))", Some "lambda path: open(path).read()")
| _ -> (None, None, None, None)
in
{
s_format = format;
s_writer = not_implemented_builtin format;
s_reader = not_implemented_builtin format;
s_r_writer = r_w;
s_r_reader = r_r;
s_py_writer = py_w;
s_py_reader = py_r;
}
in
List.iter (fun name -> register name (mk_ser name))
["csv"; "arrow"; "json"; "pmml"; "onnx"; "tlang"; "bin"; "text"]