BioImager  4.9.0
A .NET microscopy imaging application based on Bio library. Supports various microscopes by using imported libraries & GUI automation. Supports XInput game controllers to move stage, take images, run ImageJ macros on images or Bio C# scripts.
Loading...
Searching...
No Matches
ML.cs
1using System;
2using System.Collections.Generic;
3using System.IO;
4using System.Linq;
5using AForge;
6using Microsoft.ML.OnnxRuntime;
7using Microsoft.ML.OnnxRuntime.Tensors;
8using TorchSharp;
9using MathNet.Numerics.Statistics;
10using YamlDotNet;
11using YamlDotNet.Serialization.NamingConventions;
12using YamlDotNet.Serialization;
13using System.Collections;
14using Bitmap = AForge.Bitmap;
15using BioLib;
16namespace BioImager.ML
17{
18 public static class ML
19 {
20 public class Model
21 {
22 public Model(string file)
23 {
24 File = file;
25 Name = Path.GetFileNameWithoutExtension(file);
26 if (file.EndsWith(".onnx"))
27 {
28 InferenceSession = new InferenceSession(file);
29 foreach (var m in InferenceSession.InputMetadata)
30 {
31 InputValueTypes.Add(m.Value);
32 }
33 foreach (var m in InferenceSession.InputNames)
34 {
35 InputValueNames.Add(m);
36 }
37 foreach (var m in InferenceSession.OutputMetadata)
38 {
39 OutputValueTypes.Add(m.Value);
40 }
41 }
42 else if (file.EndsWith(".pt"))
43 {
44 try
45 {
46 Module = torch.jit.load(file);
47 Module.eval();
48 var chs = Module.named_modules();
49 var input = chs.First();
50 InputModule = input.module;
51 var output = chs.Last();
52 OutputModule = output.module;
53
54
55 }
56 catch (Exception e)
57 {
58 Console.WriteLine(e.Message.ToString());
59 }
60 string f = Path.GetDirectoryName(file) + "/" + Path.GetFileNameWithoutExtension(file) + ".yaml";
61 if (!System.IO.File.Exists(f))
62 {
63 MessageBox.Show("No corresponding model Yaml metadata file for:" + Name);
64 return;
65 }
66 using (var reader = new StreamReader(f))
67 {
68 var dynamicDeserializer = new DeserializerBuilder().Build();
69 Metadata = dynamicDeserializer.Deserialize<Dictionary<string, object>>(reader);
70 IDictionary conf = (IDictionary)Metadata["config"];
71 Object inpp = Metadata["inputs"];
72 List<Object> inpps = (List<Object>)inpp;
73 IDictionary inpinfo = (IDictionary)inpps[0];
74 IDictionary imagej = (IDictionary)conf["deepimagej"];
75 IDictionary test = (IDictionary)imagej["test_information"];
76 Object inputs = test["inputs"];
77 List<Object> inps = (List<Object>)inputs;
78 IDictionary info = (IDictionary)inps[0];
79 IDictionary inputinfo = (IDictionary)inpps[0];
80 IDictionary shapeinfo = (IDictionary)inputinfo["shape"];
81 List<Object> shapeinfomin = (List<Object>)shapeinfo["min"];
82 int len = shapeinfomin.Count;
83 string size = (string)info["size"];
84 string[] sts = size.Split("x");
85 if (len != sts.Length)
86 {
87 shape = new long[len];
88 for (int i = 0; i < len; i++)
89 {
90 if (len - i > sts.Length)
91 {
92 shape[i] = 1;
93 }
94 else
95 {
96 shape[i] = long.Parse(sts[sts.Length - i]);
97 }
98 }
99 }
100 else
101 {
102 shape = new long[sts.Length];
103 for (int i = 0; i < sts.Length; i++)
104 {
105 String s = sts[i];
106 shape[i] = long.Parse(s);
107 }
108 }
109 List<Object> val = (List<Object>)inpinfo["data_range"];
110 MaxValue = int.Parse(val[1].ToString());
111 }
112 }
113 }
114 public Dictionary<string, object> Metadata { get; set; } = null;
115 public InferenceSession InferenceSession { get; set; }
116 public torch.jit.ScriptModule Module { get; set; }
117 public torch.nn.Module InputModule { get; set; }
118 public torch.nn.Module OutputModule { get; set; }
119 private List<NodeMetadata> InputValueTypes { get; set; } = new List<NodeMetadata>();
120 private List<NodeMetadata> OutputValueTypes { get; set; } = new List<NodeMetadata>();
121 private List<string> InputValueNames { get; set; } = new List<string>();
122 public string File { get; set; }
123 public string Name { get; set; }
124 public int Width
125 {
126 get
127 {
128 if (File.EndsWith(".onnx"))
129 {
130 if (InputValueTypes.Count > 0)
131 {
132 int w = InputValueTypes[0].Dimensions[2];
133 if (w == 0)
134 {
135 if (OutputValueTypes.Count > 0)
136 {
137 for (int i = 0; i < OutputValueTypes.Count; i++)
138 {
139 int max = int.MinValue;
140 for (int j = 0; j < OutputValueTypes[i].Dimensions.Length; j++)
141 {
142 if (OutputValueTypes[i].Dimensions[j] > max)
143 max = OutputValueTypes[i].Dimensions[j];
144 }
145 return max;
146 }
147 }
148 }
149 return w;
150 }
151 return 0;
152 }
153 else
154 {
155 if (shape.Length == 4)
156 return (int)Shape[0];
157 else
158 return (int)Shape[4];
159 }
160 }
161 }
162 public int Height
163 {
164 get
165 {
166 if (File.EndsWith(".onnx"))
167 {
168 if (InputValueTypes.Count > 0)
169 {
170 int h = InputValueTypes[0].Dimensions[3];
171 if (h == 0)
172 {
173 if (OutputValueTypes.Count > 0)
174 {
175 for (int i = 0; i < OutputValueTypes.Count; i++)
176 {
177 int max = int.MinValue;
178 for (int j = 0; j < OutputValueTypes[i].Dimensions.Length; j++)
179 {
180 if (OutputValueTypes[i].Dimensions[j] > max)
181 max = OutputValueTypes[i].Dimensions[j];
182 }
183 return max;
184 }
185 }
186 }
187 return h;
188 }
189 return 0;
190 }
191 else
192 {
193 if (shape.Length == 4)
194 return (int)Shape[1];
195 else
196 return (int)Shape[3];
197 }
198 }
199 }
200 public int Depth
201 {
202 get
203 {
204 if (shape.Length == 4)
205 return (int)Shape[2];
206 else
207 return (int)shape[1];
208 }
209 }
210 long[] shape;
211 public long[] Shape
212 {
213 get
214 {
215 if (Metadata == null)
216 return new long[] { 1, 3, 256, 256 };
217 else
218 {
219 return shape;
220 }
221 }
222 }
223 public int MaxValue { get; set; }
224 public ModelOutputType ModelType { get; set; }
225 public enum ModelOutputType
226 {
227 classification,
228 image
229 }
230 public void Run(BioImage b)
231 {
232 if (File.EndsWith(".onnx"))
233 RunONNX(b);
234 else if (File.EndsWith(".pt"))
235 RunTorch(b);
236 }
237 public void RunONNX(BioImage b)
238 {
239 BioImage bb = new BioImage("output.ome.tif");
240 if (b.Type != BioImage.ImageType.pyramidal)
241 {
242 for (int i = 0; i < b.Buffers.Count; i++)
243 {
244 Bitmap bm = ResizeBilinear(b.Buffers[i], Width, Height);
245 int w = Width;
246 int h = Height;
247 int d = InputValueTypes[0].Dimensions[1];
248 float[] img = new float[w * h * d];
249 for (int y = 0; y < h; y++)
250 {
251 for (int x = 0; x < w; x++)
252 {
253 for (int c = 0; c < d; c++)
254 {
255 int index = (y * w * d) + x;
256 if (c > bm.RGBChannelsCount)
257 {
258 img[index] = 0;
259 }
260 else
261 {
262 if (bm.BitsPerPixel > 8)
263 img[index] = bm.GetValue(x, y, c);
264 else
265 img[index] = bm.GetValue(x, y, c);
266 }
267 }
268 }
269 }
270 var tensor = new DenseTensor<float>(img, new[] { 1, InputValueTypes[0].Dimensions[1], w, h });
271 var inputs = new List<NamedOnnxValue>
272 {
273 NamedOnnxValue.CreateFromTensor(InputValueNames[0], tensor)
274 };
275 var results = InferenceSession.Run(inputs);
276 int r = 0;
277 foreach (var result in results)
278 {
279 if (OutputValueTypes[r].Dimensions[3] == Width)
280 {
281 float[] outputmask = result.AsTensor<float>().ToArray();
282 double mean = outputmask.Mean();
283 for (int c = 0; c < OutputValueTypes[r].Dimensions[1]; c++)
284 {
285 Bitmap bmp = new Bitmap(Width, Height, b.Buffers[0].PixelFormat);
286 for (int y = 0; y < h; y++)
287 {
288 for (int x = 0; x < w; x++)
289 {
290 int index = (y * w * d) + x;
291 bmp.SetPixel(x, y, new ColorS((ushort)(outputmask[index] / mean)));
292 }
293 }
294 bb.Buffers.Add(ResizeBilinear(bmp, b.SizeX, b.SizeY));
295 }
296 }
297 r++;
298 }
299 }
300 bb.Channels.AddRange(b.Channels);
301 if (OutputValueTypes[0].Dimensions[1] > 1)
302 {
303 bb.Channels.AddRange(b.Channels);
304 for (int i = 0; i < bb.Channels.Count; i++)
305 {
306 bb.Channels[i].Index = i;
307 }
308 }
309 bb.UpdateCoords(b.SizeZ, b.SizeC * OutputValueTypes[0].Dimensions[1], b.SizeT);
310 bb.Resolutions.Add(new Resolution(bb.SizeX, bb.SizeY, bb.Buffers[0].PixelFormat, b.PhysicalSizeX, b.PhysicalSizeY, b.PhysicalSizeZ, b.StageSizeX, b.StageSizeY, b.StageSizeZ));
311 bb.Volume = b.Volume;
312 bb.bitsPerPixel = bb.Buffers[0].BitsPerPixel;
313
314 BioImage.AutoThreshold(bb, true);
315 Images.AddImage(bb);
316 App.tabsView.AddTab(bb);
317 }
318 else
319 throw new NotImplementedException();
320 }
321 public void RunTorch(BioImage b)
322 {
323 BioImage bb = new BioImage("output.ome.tif");
324 if (b.Type != BioImage.ImageType.pyramidal)
325 {
326 int r = 0;
327 int resDepth = 1;
328 if (shape.Length == 4)
329 {
330 for (int i = 0; i < b.Buffers.Count; i++)
331 {
332 Bitmap bm = ResizeBilinear(b.Buffers[i], Width, Height);
333 int w = Width;
334 int h = Height;
335 int d = Depth;
336 float[,,,] img = new float[1, Depth, Width, Height];
337 for (int y = 0; y < h; y++)
338 {
339 for (int x = 0; x < w; x++)
340 {
341 for (int c = 0; c < d; c++)
342 {
343 int index = (y * w * c) + x;
344 if (c >= bm.RGBChannelsCount)
345 {
346 img[0, c, x, y] = 0;
347 }
348 else
349 {
350 if (bm.BitsPerPixel > 8)
351 img[0, c, x, y] = ((float)bm.GetValue(x, y, c) / (float)ushort.MaxValue) * MaxValue;
352 else
353 img[0, c, x, y] = ((float)bm.GetValue(x, y, c) / (float)byte.MaxValue) * MaxValue;
354 }
355 }
356 }
357 }
358 var tensor = torch.tensor(img);
359 object results = null;
360 try
361 {
362 results = Module.forward(tensor);
363 }
364 catch (Exception e)
365 {
366 Console.WriteLine(e.ToString());
367 }
368 TorchSharp.Utils.TensorAccessor<float> outputmask = ((torch.Tensor)results).data<float>();
369 double mean = outputmask.Mean();
370 resDepth = (int)((torch.Tensor)results).shape[1];
371 for (int c = 0; c < resDepth; c++)
372 {
373 Bitmap bmp = new Bitmap(Width, Height, b.Buffers[0].PixelFormat);
374 for (int y = 0; y < h; y++)
375 {
376 for (int x = 0; x < w; x++)
377 {
378 int index = (y * w * c) + x;
379 bmp.SetPixel(x, y, new ColorS((ushort)(outputmask[index] / mean)));
380 }
381 }
382 bb.Buffers.Add(ResizeBilinear(bmp, b.SizeX, b.SizeY));
383 bb.Buffers[bb.Buffers.Count - 1].RotateFlip(AForge.RotateFlipType.Rotate90FlipX);
384 }
385 r++;
386
387 }
388 }
389 else if (shape.Length == 5)
390 {
391 torch.Tensor tensor = torch.empty(shape);
392 for (int i = 0; i < b.Buffers.Count; i++)
393 {
394 Bitmap bm = ResizeBilinear(b.Buffers[i], Width, Height);
395 for (int y = 0; y < Height; y++)
396 {
397 for (int x = 0; x < Width; x++)
398 {
399 for (int c = 0; c < Depth; c++)
400 {
401 if (c >= bm.RGBChannelsCount)
402 {
403 tensor[0, c, i, x, y] = 0;
404 }
405 else
406 {
407 if (bm.BitsPerPixel > 8)
408 tensor[0, c, i, x, y] = ((float)bm.GetValue(x, y, c) / (float)ushort.MaxValue) * MaxValue;
409 else
410 tensor[0, c, i, x, y] = ((float)bm.GetValue(x, y, c) / (float)byte.MaxValue) * MaxValue;
411 }
412 }
413 }
414 }
415 }
416 torch.Tensor outputmask = (torch.Tensor)Module.forward(tensor);
417 resDepth = (int)(outputmask.shape[1]);
418 for (int c = 0; c < resDepth; c++)
419 {
420 for (int buf = 0; buf < (int)outputmask.shape[2]; buf++)
421 {
422 Bitmap bmp = new Bitmap((int)outputmask.shape[3], (int)outputmask.shape[4], b.Buffers[0].PixelFormat);
423 for (int y = 0; y < Height; y++)
424 {
425 for (int x = 0; x < Width; x++)
426 {
427 float by = outputmask[0, c, buf, x, y].ToSingle();
428 bmp.SetPixel(x, y, new ColorS((ushort)(by * byte.MaxValue)));
429 }
430 }
431 bb.Buffers.Add(ResizeBilinear(bmp, b.SizeX, b.SizeY));
432 bb.Buffers[bb.Buffers.Count - 1].RotateFlip(AForge.RotateFlipType.Rotate90FlipX);
433 }
434 }
435 }
436 bb.Channels.AddRange(b.Channels);
437 if (resDepth > 1)
438 {
439 bb.Channels.AddRange(b.Channels);
440 for (int i = 0; i < bb.Channels.Count; i++)
441 {
442 bb.Channels[i].Index = i;
443 }
444 }
445 bb.UpdateCoords(b.SizeZ, b.SizeC * resDepth, b.SizeT);
446 bb.Resolutions.Add(new Resolution(bb.SizeX, bb.SizeY, bb.Buffers[0].PixelFormat, b.PhysicalSizeX, b.PhysicalSizeY, b.PhysicalSizeZ, b.StageSizeX, b.StageSizeY, b.StageSizeZ));
447 bb.Volume = b.Volume;
448 bb.bitsPerPixel = bb.Buffers[0].BitsPerPixel;
449 BioImage.AutoThreshold(bb, true);
450 Images.AddImage(bb);
451 App.tabsView.AddTab(bb);
452 }
453 else
454 throw new NotImplementedException();
455 }
456 }
457 public static List<Model> Models = new List<Model>();
458 public static void Load(string mod)
459 {
460 Models.Add(new Model(mod));
461 }
462
463 public static void Run(string modelsName, BioImage image)
464 {
465 string st = System.IO.Path.GetDirectoryName(Environment.ProcessPath) + "/Models/" + modelsName;
466 bool loaded = false;
467 foreach (var m in Models)
468 {
469 if (m.Name == Path.GetFileNameWithoutExtension(modelsName))
470 {
471 loaded = true;
472 break;
473 }
474 }
475 if (!loaded)
476 {
477 Models.Add(new Model(st));
478 }
479 foreach (var m in Models)
480 {
481 if (m.Name == Path.GetFileNameWithoutExtension(modelsName))
482 {
483 m.Run(image);
484 }
485 }
486 }
487 public static Bitmap ResizeBilinear(Bitmap original, int targetWidth, int targetHeight)
488 {
489 // Create a new Pixbuf with target dimensions
490 Bitmap resized = new Bitmap(targetWidth, targetHeight, original.PixelFormat);
491
492 int originalWidth = original.Width;
493 int originalHeight = original.Height;
494 float xRatio = originalWidth / (float)targetWidth;
495 float yRatio = originalHeight / (float)targetHeight;
496
497 // Lock the bits of the original and resized images (if necessary)
498
499 for (int y = 0; y < targetHeight; y++)
500 {
501 for (int x = 0; x < targetWidth; x++)
502 {
503 float gx = ((float)x) * xRatio;
504 float gy = ((float)y) * yRatio;
505 int gxi = (int)gx;
506 int gyi = (int)gy;
507
508 ColorS topLeft = original.GetPixel(gxi, gyi);
509 ColorS topRight = original.GetPixel(gxi + 1, gyi);
510 ColorS bottomLeft = original.GetPixel(gxi, gyi + 1);
511 ColorS bottomRight = original.GetPixel(gxi + 1, gyi + 1);
512
513 ushort red = (ushort)BilinearInterpolate(gx - gxi, gy - gyi, topLeft.R, topRight.R, bottomLeft.R, bottomRight.R);
514 ushort green = (ushort)BilinearInterpolate(gx - gxi, gy - gyi, topLeft.G, topRight.G, bottomLeft.G, bottomRight.G);
515 ushort blue = (ushort)BilinearInterpolate(gx - gxi, gy - gyi, topLeft.B, topRight.B, bottomLeft.B, bottomRight.B);
516
517 resized.SetPixel(x, y, new ColorS(red, green, blue));
518 }
519 }
520
521 // Unlock the bits (if necessary)
522
523 return resized;
524 }
525 private static float BilinearInterpolate(float x, float y, float topLeft, float topRight, float bottomLeft, float bottomRight)
526 {
527 float top = Interpolate(x, topLeft, topRight);
528 float bottom = Interpolate(x, bottomLeft, bottomRight);
529 return Interpolate(y, top, bottom);
530 }
531
532 private static float Interpolate(float t, float a, float b)
533 {
534 return a + (b - a) * t;
535 }
536
537 // Implement GetPixel and SetPixel methods based on how Gdk.Pixbuf manages pixel data
538
539 public static void Initialize()
540 {
541 string st = System.IO.Path.GetDirectoryName(Environment.ProcessPath);
542 Directory.CreateDirectory(st + "/Models");
543 foreach (string f in Directory.GetFiles(st + "/Models"))
544 {
545 if (f.EndsWith(".onnx") || f.EndsWith(".pt"))
546 {
547 string path = "Run/" + Path.GetFileName(f);
548 App.AddMenu(path);
549 }
550 }
551 }
552 }
553}
void AddTab(BioImage b)
Definition TabsView.cs:129