BioGTK  6.7.0
A .NET library & program for annotating, editing various microscopy imaging formats using Bioformats supported images. Including whole slide, pyramidal, and series.
Loading...
Searching...
No Matches
ML.cs
1using System;
2using System.Collections.Generic;
3using System.IO;
4using System.Linq;
5using AForge;
6using Microsoft.ML.OnnxRuntime;
7using Microsoft.ML.OnnxRuntime.Tensors;
8using TorchSharp;
9using MathNet.Numerics.Statistics;
10using YamlDotNet;
11using YamlDotNet.Serialization.NamingConventions;
12using YamlDotNet.Serialization;
13using System.Collections;
14using Bitmap = AForge.Bitmap;
15
16namespace BioGTK.ML
17{
18 public static class ML
19 {
20 public class Model
21 {
22 public Model(string file)
23 {
24 File = file;
25 Name = Path.GetFileNameWithoutExtension(file);
26 string dir = System.IO.Path.GetDirectoryName(Environment.ProcessPath);
27 File = dir.Replace("\\","/") + "/Models/" + file;
28 if (file.EndsWith(".onnx"))
29 {
30 InferenceSession = new InferenceSession(file);
31 foreach (var m in InferenceSession.InputMetadata)
32 {
33 InputValueTypes.Add(m.Value);
34 }
35 foreach (var m in InferenceSession.InputNames)
36 {
37 InputValueNames.Add(m);
38 }
39 foreach (var m in InferenceSession.OutputMetadata)
40 {
41 OutputValueTypes.Add(m.Value);
42 }
43 }
44 else if (file.EndsWith(".pt"))
45 {
46 try
47 {
48 //torch.load(File);
49 Module = torch.jit.load(File);
50 Module.eval();
51 var chs = Module.named_modules();
52 var input = chs.First();
53 InputModule = input.module;
54 var output = chs.Last();
55 OutputModule = output.module;
56 }
57 catch (Exception e)
58 {
59 Console.WriteLine("Failed to run pytorch model. Ensure that PyTorch is properly installed.");
60 Console.WriteLine(e.Message.ToString());
61 }
62 string f = System.IO.Path.GetDirectoryName(System.Environment.ProcessPath) + "/Models/" + Path.GetFileNameWithoutExtension(file) + ".yaml";
63 if (!System.IO.File.Exists(f))
64 {
65 Gtk.MessageDialog md = new Gtk.MessageDialog(App.tabsView, Gtk.DialogFlags.Modal, Gtk.MessageType.Error, Gtk.ButtonsType.Ok,"No corresponding model Yaml metadata file for:" + Name);
66 md.Show();
67 md.Run();
68 return;
69 }
70 using (var reader = new StreamReader(f))
71 {
72 var dynamicDeserializer = new DeserializerBuilder().Build();
73 Metadata = dynamicDeserializer.Deserialize<Dictionary<string, object>>(reader);
74 IDictionary conf = (IDictionary)Metadata["config"];
75 Object inpp = Metadata["inputs"];
76 List<Object> inpps = (List<Object>)inpp;
77 IDictionary inpinfo = (IDictionary)inpps[0];
78 IDictionary imagej = (IDictionary)conf["deepimagej"];
79 IDictionary test = (IDictionary)imagej["test_information"];
80 Object inputs = test["inputs"];
81 List<Object> inps = (List<Object>)inputs;
82 IDictionary info = (IDictionary)inps[0];
83 IDictionary inputinfo = (IDictionary)inpps[0];
84 IDictionary shapeinfo = (IDictionary)inputinfo["shape"];
85 if (shapeinfo != null)
86 {
87 List<Object> shapeinfomin = (List<Object>)shapeinfo["min"];
88 int len = shapeinfomin.Count;
89 string size = (string)info["size"];
90 string[] sts = size.Split("x");
91 if (len != sts.Length)
92 {
93 shape = new long[len];
94 for (int i = 0; i < len; i++)
95 {
96 if (len - i > sts.Length)
97 {
98 shape[i] = 1;
99 }
100 else
101 {
102 shape[i] = long.Parse(sts[sts.Length - i]);
103 }
104 }
105 }
106 else
107 {
108 shape = new long[sts.Length];
109 for (int i = 0; i < sts.Length; i++)
110 {
111 String s = sts[i];
112 shape[i] = long.Parse(s);
113 }
114 }
115 List<Object> val = (List<Object>)inpinfo["data_range"];
116 MaxValue = int.Parse(val[1].ToString());
117 }
118 else
119 {
120 string sh = (string)info["size"];
121 string[] sts = (string[])sh.Split("x");
122 IList inp = (IList)inputinfo["axes"];
123 int co = inp.Count;
124 shape = new long[co];
125 for (int i = 0; i < co; i++)
126 {
127 IDictionary dict = (IDictionary)inp[i];
128 string type = (string)dict["type"];
129 string id = (string)dict["id"];
130 if (type == "batch")
131 {
132 shape[co - 5] = 1;
133 }
134 if (type == "channel")
135 {
136 shape[co - 4] = long.Parse(sts[3]);
137 }
138 if (id == "z")
139 {
140 shape[co - 3] = long.Parse(sts[2]);
141 }
142 if (id == "y")
143 {
144 shape[co - 2] = long.Parse(sts[1]);
145 }
146 if (id == "x")
147 {
148 shape[co - 1] = long.Parse(sts[0]);
149 }
150 }
151 MaxValue = 256;
152 }
153 }
154 }
155 BioLib.Recorder.AddLine("BioGTK.ML.Model model = new BioGTK.ML.Model(\"" + file + "\");",false);
156 }
157 public Dictionary<string, object> Metadata { get; set; } = null;
158 public InferenceSession InferenceSession { get; set; }
159 public torch.jit.ScriptModule Module { get; set; }
160 public torch.nn.Module InputModule { get; set; }
161 public torch.nn.Module OutputModule { get; set; }
162 private List<NodeMetadata> InputValueTypes { get; set; } = new List<NodeMetadata>();
163 private List<NodeMetadata> OutputValueTypes { get; set; } = new List<NodeMetadata>();
164 private List<string> InputValueNames { get; set; } = new List<string>();
165 public string File { get; set; }
166 public string Name { get; set; }
167 public int Width
168 {
169 get
170 {
171 if (File.EndsWith(".onnx"))
172 {
173 if (InputValueTypes.Count > 0)
174 {
175 int w = InputValueTypes[0].Dimensions[2];
176 if (w == 0)
177 {
178 if (OutputValueTypes.Count > 0)
179 {
180 for (int i = 0; i < OutputValueTypes.Count; i++)
181 {
182 int max = int.MinValue;
183 for (int j = 0; j < OutputValueTypes[i].Dimensions.Length; j++)
184 {
185 if (OutputValueTypes[i].Dimensions[j] > max)
186 max = OutputValueTypes[i].Dimensions[j];
187 }
188 return max;
189 }
190 }
191 }
192 return w;
193 }
194 return 0;
195 }
196 else
197 {
198 if (shape.Length == 4)
199 return (int)Shape[0];
200 else
201 return (int)Shape[4];
202 }
203 }
204 }
205 public int Height
206 {
207 get
208 {
209 if (File.EndsWith(".onnx"))
210 {
211 if (InputValueTypes.Count > 0)
212 {
213 int h = InputValueTypes[0].Dimensions[3];
214 if (h == 0)
215 {
216 if (OutputValueTypes.Count > 0)
217 {
218 for (int i = 0; i < OutputValueTypes.Count; i++)
219 {
220 int max = int.MinValue;
221 for (int j = 0; j < OutputValueTypes[i].Dimensions.Length; j++)
222 {
223 if (OutputValueTypes[i].Dimensions[j] > max)
224 max = OutputValueTypes[i].Dimensions[j];
225 }
226 return max;
227 }
228 }
229 }
230 return h;
231 }
232 return 0;
233 }
234 else
235 {
236 if (shape.Length == 4)
237 return (int)Shape[1];
238 else
239 return (int)Shape[3];
240 }
241 }
242 }
243 public int Depth
244 {
245 get
246 {
247 if (shape.Length == 4)
248 return (int)Shape[2];
249 else
250 return (int)shape[1];
251 }
252 }
253 long[] shape;
254 public long[] Shape
255 {
256 get
257 {
258 if (Metadata == null)
259 return new long[] { 1, 3, 256, 256 };
260 else
261 {
262 return shape;
263 }
264 }
265 }
266 public int MaxValue { get; set; }
267 public ModelOutputType ModelType { get; set; }
268 public enum ModelOutputType
269 {
270 classification,
271 image
272 }
273 public void Run(BioImage b)
274 {
275 if (File.EndsWith(".onnx"))
276 RunONNX(b);
277 else if (File.EndsWith(".pt"))
278 RunTorch(b);
279 BioLib.Recorder.AddLine("model.Run(Images.GetImage(\"" + b.Filename + "\");",false);
280 }
281 public void RunONNX(BioImage b)
282 {
283 BioImage bb = new BioImage("output.ome.tif");
284 if (b.Type != BioImage.ImageType.pyramidal)
285 {
286 for (int i = 0; i < b.Buffers.Count; i++)
287 {
288 Bitmap bm = ResizeBilinear(b.Buffers[i], Width, Height);
289 int w = Width;
290 int h = Height;
291 int d = InputValueTypes[0].Dimensions[1];
292 float[] img = new float[w * h * d];
293 for (int y = 0; y < h; y++)
294 {
295 for (int x = 0; x < w; x++)
296 {
297 for (int c = 0; c < d; c++)
298 {
299 int index = (y * w * d) + x;
300 if (c > bm.RGBChannelsCount)
301 {
302 img[index] = 0;
303 }
304 else
305 {
306 if (bm.BitsPerPixel > 8)
307 img[index] = bm.GetValue(x, y, c);
308 else
309 img[index] = bm.GetValue(x, y, c);
310 }
311 }
312 }
313 }
314 var tensor = new DenseTensor<float>(img, new[] { 1, InputValueTypes[0].Dimensions[1], w, h });
315 var inputs = new List<NamedOnnxValue>
316 {
317 NamedOnnxValue.CreateFromTensor(InputValueNames[0], tensor)
318 };
319 var results = InferenceSession.Run(inputs);
320 int r = 0;
321 foreach (var result in results)
322 {
323 if (OutputValueTypes[r].Dimensions[3] == Width)
324 {
325 float[] outputmask = result.AsTensor<float>().ToArray();
326 double mean = outputmask.Mean();
327 for (int c = 0; c < OutputValueTypes[r].Dimensions[1]; c++)
328 {
329 Bitmap bmp = new Bitmap(Width, Height, b.Buffers[0].PixelFormat);
330 for (int y = 0; y < h; y++)
331 {
332 for (int x = 0; x < w; x++)
333 {
334 int index = (y * w * d) + x;
335 bmp.SetPixel(x, y, new ColorS((ushort)(outputmask[index] / mean)));
336 }
337 }
338 bb.Buffers.Add(ResizeBilinear(bmp, b.SizeX, b.SizeY));
339 }
340 }
341 r++;
342 }
343 }
344 bb.Channels.AddRange(b.Channels);
345 if (OutputValueTypes[0].Dimensions[1] > 1)
346 {
347 bb.Channels.AddRange(b.Channels);
348 for (int i = 0; i < bb.Channels.Count; i++)
349 {
350 bb.Channels[i].Index = i;
351 }
352 }
353 bb.UpdateCoords(b.SizeZ, b.SizeC * OutputValueTypes[0].Dimensions[1], b.SizeT);
354 bb.Resolutions.Add(new Resolution(bb.SizeX, bb.SizeY, bb.Buffers[0].PixelFormat, b.PhysicalSizeX, b.PhysicalSizeY, b.PhysicalSizeZ, b.StageSizeX, b.StageSizeY, b.StageSizeZ));
355 bb.Volume = b.Volume;
356 BioImage.AutoThreshold(bb, true);
357 Images.AddImage(bb);
358 }
359 else
360 throw new NotImplementedException();
361 }
362 public void RunTorch(BioImage b)
363 {
364 BioImage bb = new BioImage("output.ome.tif");
365 if (b.Type != BioImage.ImageType.pyramidal)
366 {
367 int r = 0;
368 int resDepth = 1;
369 if (shape.Length == 4)
370 {
371 for (int i = 0; i < b.Buffers.Count; i++)
372 {
373 Bitmap bm = ResizeBilinear(b.Buffers[i], Width, Height);
374 int w = Width;
375 int h = Height;
376 int d = Depth;
377 float[,,,] img;
378 if (shape[0] > shape[3])
379 {
380 img = new float[1, Depth, Width, Height];
381 }
382 else
383 {
384 img = new float[Width, Height, Depth, 1];
385 }
386 for (int y = 0; y < h; y++)
387 {
388 for (int x = 0; x < w; x++)
389 {
390 for (int c = 0; c < d; c++)
391 {
392 int index = (y * w * c) + x;
393 if (c >= bm.RGBChannelsCount)
394 {
395 img[0, c, x, y] = 0;
396 }
397 else
398 {
399 if (bm.BitsPerPixel > 8)
400 img[0, c, x, y] = ((float)bm.GetValue(x, y, c) / (float)ushort.MaxValue) * MaxValue;
401 else
402 img[0, c, x, y] = ((float)bm.GetValue(x, y, c) / (float)byte.MaxValue) * MaxValue;
403 }
404 }
405 }
406 }
407 var tensor = torch.tensor(img);
408 object results = null;
409 try
410 {
411 results = Module.forward(tensor);
412 }
413 catch (Exception e)
414 {
415 Console.WriteLine(e.ToString());
416 }
417 TorchSharp.Utils.TensorAccessor<float> outputmask = ((torch.Tensor)results).data<float>();
418 double mean = outputmask.Mean();
419 resDepth = (int)((torch.Tensor)results).shape[1];
420 for (int c = 0; c < resDepth; c++)
421 {
422 Bitmap bmp = new Bitmap(Width, Height, b.Buffers[0].PixelFormat);
423 for (int y = 0; y < h; y++)
424 {
425 for (int x = 0; x < w; x++)
426 {
427 int index = (y * w * c) + x;
428 bmp.SetPixel(x, y, new ColorS((ushort)(outputmask[index] / mean)));
429 }
430 }
431 bb.Buffers.Add(ResizeBilinear(bmp, b.SizeX, b.SizeY));
432 bb.Buffers[bb.Buffers.Count - 1].RotateFlip(AForge.RotateFlipType.Rotate90FlipX);
433 }
434 r++;
435
436 }
437 }
438 else if (shape.Length == 5)
439 {
440 torch.Tensor tensor = torch.empty(shape);
441 for (int i = 0; i < shape[2]; i++)
442 {
443 Bitmap bm = ResizeBilinear(b.Buffers[i], Width, Height);
444 for (int y = 0; y < Height; y++)
445 {
446 for (int x = 0; x < Width; x++)
447 {
448 for (int c = 0; c < Depth; c++)
449 {
450 if (c >= bm.RGBChannelsCount)
451 {
452 tensor[0, c, i, x, y] = 0;
453 }
454 else
455 {
456 if (bm.BitsPerPixel > 8)
457 tensor[0, c, i, x, y] = ((float)bm.GetValue(x, y, c) / (float)ushort.MaxValue) * MaxValue;
458 else
459 tensor[0, c, i, x, y] = ((float)bm.GetValue(x, y, c) / (float)byte.MaxValue) * MaxValue;
460 }
461 }
462 }
463 }
464 }
465 torch.Tensor outputmask = (torch.Tensor)Module.forward(tensor);
466 resDepth = (int)(outputmask.shape[1]);
467 for (int c = 0; c < resDepth; c++)
468 {
469 for (int buf = 0; buf < (int)outputmask.shape[2]; buf++)
470 {
471 Bitmap bmp = new Bitmap((int)outputmask.shape[3], (int)outputmask.shape[4], b.Buffers[0].PixelFormat);
472 for (int y = 0; y < Height; y++)
473 {
474 for (int x = 0; x < Width; x++)
475 {
476 float by = outputmask[0, c, buf, x, y].ToSingle();
477 bmp.SetPixel(x, y, new ColorS((ushort)(by * byte.MaxValue)));
478 }
479 }
480 bb.Buffers.Add(ResizeBilinear(bmp, b.SizeX, b.SizeY));
481 bb.Buffers[bb.Buffers.Count - 1].RotateFlip(AForge.RotateFlipType.Rotate90FlipX);
482 }
483 }
484 }
485 bb.Channels.AddRange(b.Channels);
486 if (resDepth > 1)
487 {
488 bb.Channels.AddRange(b.Channels);
489 for (int i = 0; i < bb.Channels.Count; i++)
490 {
491 bb.Channels[i].Index = i;
492 }
493 }
494 bb.UpdateCoords(b.SizeZ, b.SizeC * resDepth, b.SizeT);
495 bb.Resolutions.Add(new Resolution(bb.SizeX, bb.SizeY, bb.Buffers[0].PixelFormat, b.PhysicalSizeX, b.PhysicalSizeY, b.PhysicalSizeZ, b.StageSizeX, b.StageSizeY, b.StageSizeZ));
496 bb.Volume = b.Volume;
497 BioImage.AutoThreshold(bb, true);
498 App.tabsView.AddTab(bb);
499 }
500 else
501 throw new NotImplementedException();
502 }
503 }
504 public static List<Model> Models = new List<Model>();
505 public static void Load(string mod)
506 {
507 Models.Add(new Model(mod));
508 }
509
510 public static void Run(string modelsName, BioImage image)
511 {
512 bool loaded = false;
513 foreach (var item in Models)
514 {
515 if (item.File.Contains(image.Filename))
516 loaded = true;
517 }
518 if (!loaded)
519 Load(modelsName);
520 string st = System.IO.Path.GetDirectoryName(Environment.ProcessPath) + "/Models/" + modelsName;
521 foreach (var m in Models)
522 {
523 if (m.Name == Path.GetFileNameWithoutExtension(modelsName))
524 {
525 loaded = true;
526 break;
527 }
528 }
529 if(!loaded)
530 {
531 Models.Add(new Model(st));
532 }
533 foreach (var m in Models)
534 {
535 if (m.Name == Path.GetFileNameWithoutExtension(modelsName))
536 {
537 m.Run(image);
538 }
539 }
540 }
541 public static Bitmap ResizeBilinear(Bitmap original, int targetWidth, int targetHeight)
542 {
543 AForge.Imaging.Filters.ResizeBilinear re = new AForge.Imaging.Filters.ResizeBilinear(targetWidth, targetHeight);
544 return re.Apply(original);
545 }
546
547 public static void Initialize()
548 {
549 string st = System.IO.Path.GetDirectoryName(Environment.ProcessPath);
550 foreach (string f in Directory.GetFiles(st + "/Models"))
551 {
552 if (f.EndsWith(".onnx") || f.EndsWith(".pt"))
553 {
554 string path = "Run/" + Path.GetFileName(f);
555 App.AddMenu(path);
556 }
557 }
558 }
559 }
560}
BioLib
Definition Zarr.cs:20