BioGTK  6.7.0
A .NET library & program for annotating, editing various microscopy imaging formats using Bioformats supported images. Including whole slide, pyramidal, and series.
Loading...
Searching...
No Matches
SAM.cs
1using AForge;
2using BioGTK;
3using Microsoft.ML.OnnxRuntime;
4using Microsoft.ML.OnnxRuntime.Tensors;
5using System;
6using System.Collections.Generic;
7using System.IO;
8using System.Linq;
9using System.Text;
10using System.Threading;
11using System.Threading.Tasks;
12using System.Windows;
13using System.Diagnostics;
14using Gtk;
15
16namespace BioGTK
17{
18
19 public class SAM : IDisposable
20 {
21 public static bool SAM2 = true;
22 public static SAM theSingleton = null;
23 InferenceSession mEncoder;
24 InferenceSession mDecoder;
25 bool mReady = false;
26 public float mask_threshold = 0.0f;
27 protected SAM()
28 {
29 }
33 public static SAM Instance()
34 {
35 if (null == theSingleton)
36 {
37 theSingleton = new SAM();
38 }
39 return theSingleton;
40 }
41
43 public void LoadONNXModel()
44 {
45 if (this.mEncoder != null)
46 this.mEncoder.Dispose();
47
48 if (this.mDecoder != null)
49 this.mDecoder.Dispose();
50
51 string exePath = Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
52 string encode_model_path = exePath + "/encoder-quant.onnx";
53 this.mEncoder = new InferenceSession(encode_model_path);
54
55 string decode_model_path = exePath + "/decoder-quant.onnx";
56 this.mDecoder = new InferenceSession(decode_model_path);
57 SAMTool.Encode();
58 }
59 public void Encode()
60 {
61 Encode(ImageView.SelectedImage);
62 }
69 public void Encode(BioImage b)
70 {
71 if (b.Buffers.Count * 3 * 1024 * 1024 > 4e8)
72 {
73 // Create the message dialog
74 MessageDialog msgBox = new MessageDialog(
75 null,
76 DialogFlags.Modal,
77 MessageType.Info,
78 ButtonsType.Ok | ButtonsType.Cancel,
79 "Memory required is more than 4GB are you sure you want to continue?"
80 );
81
82 // Show the message dialog
83 if (msgBox.Run() != (int)ResponseType.Ok)
84 return;
85 }
86 Progress pr = Progress.Create("Encoding Image", "Transforming", "");
87 // update ui on main UI thread
88 Application.Invoke(delegate
89 {
90 pr.Show();
91 pr.Present();
92 });
93 int i = 0;
94 foreach (Bitmap bu in b.Buffers)
95 {
96 Transforms tranform = new Transforms(1024);
97 float[] img = tranform.ApplyImage(bu);
98 var tensor = new DenseTensor<float>(img, new[] { 1, 3, 1024, 1024 });
99 IDisposableReadOnlyCollection<DisposableNamedOnnxValue> results = null;
100 List<NamedOnnxValue> inputs = new List<NamedOnnxValue>
101 {
102 NamedOnnxValue.CreateFromTensor("image", tensor)
103 };
104 try
105 {
106 results = this.mEncoder.Run(inputs);
107 }
108 catch (Exception e)
109 {
110 Console.WriteLine("Tried running SAM-2 next trying SAM-1");
111 SAM2 = false;
112 }
113 if(!SAM2)
114 {
115 inputs = new List<NamedOnnxValue>
116 {
117 NamedOnnxValue.CreateFromTensor("x", tensor)
118 };
119 results = this.mEncoder.Run(inputs);
120 }
121 if (b.Tag == null)
122 {
123 b.Tag = new List<float[]>();
124 }
125 List<float[]> l = (List<float[]>)b.Tag;
126 if(!SAM2)
127 l.Add(results.First().AsTensor<float>().ToArray());
128 else
129 l.Add(results.Last().AsTensor<float>().ToArray());
130 b.Tag = l;
131 results.Dispose();
132 pr.ProgressValue = (double)i/b.Buffers.Count;
133 i++;
134 }
135 // update ui on main UI thread
136 Application.Invoke(delegate
137 {
138 pr.Hide();
139 });
140 this.mReady = true;
141 BioLib.Recorder.AddLine("SAM.Encode(Images.GetImage(\"" + b.Filename + "\"));", false);
142 }
143
153 public float[] Decode(BioImage b, List<Promotion> promotions, int orgWid, int orgHei)
154 {
155 ZCT c = App.viewer.GetCoordinate();
156 List<float[]> lts = (List<float[]>)b.Tag;
157 int fr = b.GetFrameIndex(c.Z, c.C, c.T);
158 var embedding_tensor = new DenseTensor<float>(lts[fr], new[] { 1, 256, 64, 64 });
159 var bpmos = promotions.FindAll(e => e.mType == PromotionType.Box);
160 var pproms = promotions.FindAll(e => e.mType == PromotionType.Point);
161 int boxCount = promotions.FindAll(e => e.mType == PromotionType.Box).Count();
162 int pointCount = promotions.FindAll(e => e.mType == PromotionType.Point).Count();
163 float[] promotion = new float[2 * (boxCount * 2 + pointCount)];
164 float[] label = new float[boxCount * 2 + pointCount];
165 for (int i = 0; i < boxCount; i++)
166 {
167 var input = bpmos[i].GetInput();
168 for (int j = 0; j < input.Count(); j++)
169 {
170 promotion[4 * i + j] = input[j];
171 }
172 var la = bpmos[i].GetLable();
173 for (int j = 0; j < la.Count(); j++)
174 {
175 label[2 * i + j] = la[j];
176 }
177 }
178 for (int i = 0; i < pointCount; i++)
179 {
180 var p = pproms[i].GetInput();
181 for (int j = 0; j < p.Count(); j++)
182 {
183 promotion[boxCount * 4 + 2 * i + j] = p[j];
184 }
185 var la = pproms[i].GetLable();
186 for (int j = 0; j < la.Count(); j++)
187 {
188 label[boxCount * 2 + i + j] = la[j];
189 }
190 }
191
192 var point_coords_tensor = new DenseTensor<float>(promotion, new[] { 1, boxCount * 2 + pointCount, 2 });
193
194 var point_label_tensor = new DenseTensor<float>(label, new[] { 1, boxCount * 2 + pointCount });
195
196 float[] mask = new float[256 * 256];
197 for (int i = 0; i < mask.Count(); i++)
198 {
199 mask[i] = 0;
200 }
201 var mask_tensor = new DenseTensor<float>(mask, new[] { 1, 1, 256, 256 });
202
203 float[] hasMaskValues = new float[1] { 0 };
204 var hasMaskValues_tensor = new DenseTensor<float>(hasMaskValues, new[] { 1 });
205
206 var decode_inputs = new List<NamedOnnxValue>();
207 if (SAM2)
208 {
209 int[] orig_im_size_values = { (int)orgHei, (int)orgWid };
210 var orig_im_size_values_tensor = new DenseTensor<int>(orig_im_size_values, new[] { 2 });
211 var fs = new float[32 * 256 * 256];
212 var feats = new DenseTensor<float>(fs, new[] { 1, 32, 256, 256 });
213 var fs2 = new float[64 * 128 * 128];
214 var feats2 = new DenseTensor<float>(fs2, new[] { 1, 64, 128, 128 });
215 decode_inputs = new List<NamedOnnxValue>
216 {
217 NamedOnnxValue.CreateFromTensor("image_embed", embedding_tensor),
218 NamedOnnxValue.CreateFromTensor("high_res_feats_0", feats),
219 NamedOnnxValue.CreateFromTensor("high_res_feats_1", feats2),
220 NamedOnnxValue.CreateFromTensor("point_coords", point_coords_tensor),
221 NamedOnnxValue.CreateFromTensor("point_labels", point_label_tensor),
222 NamedOnnxValue.CreateFromTensor("mask_input", mask_tensor),
223 NamedOnnxValue.CreateFromTensor("has_mask_input", hasMaskValues_tensor),
224 NamedOnnxValue.CreateFromTensor("orig_im_size", orig_im_size_values_tensor)
225 };
226 }
227 else
228 {
229 float[] orig_im_size_values = { (float)orgHei, (float)orgWid };
230 var orig_im_size_values_tensor = new DenseTensor<float>(orig_im_size_values, new[] { 2 });
231 decode_inputs = new List<NamedOnnxValue>
232 {
233 NamedOnnxValue.CreateFromTensor("image_embeddings", embedding_tensor),
234 NamedOnnxValue.CreateFromTensor("point_coords", point_coords_tensor),
235 NamedOnnxValue.CreateFromTensor("point_labels", point_label_tensor),
236 NamedOnnxValue.CreateFromTensor("mask_input", mask_tensor),
237 NamedOnnxValue.CreateFromTensor("has_mask_input", hasMaskValues_tensor),
238 NamedOnnxValue.CreateFromTensor("orig_im_size", orig_im_size_values_tensor)
239 };
240 }
241 var segmask = this.mDecoder.Run(decode_inputs);
242 var outputmask = segmask.First().AsTensor<float>().ToArray();
243 BioLib.Recorder.AddLine("App.samTool.sam.Decode(Images.GetImage(\"" + b.Filename + "\"),App.samTool.Promotions," + orgWid + "," + orgHei + ");", false);
244 return outputmask;
245
246 }
247
248 public MaskData Decode(List<Promotion> promotions, float[] embedding, int orgWid, int orgHei)
249 {
250 var embedding_tensor = new DenseTensor<float>(embedding, new[] { 1, 256, 64, 64 });
251
252 var bpmos = promotions.FindAll(e => e.mType == PromotionType.Box);
253 var pproms = promotions.FindAll(e => e.mType == PromotionType.Point);
254 int boxCount = promotions.FindAll(e => e.mType == PromotionType.Box).Count();
255 int pointCount = promotions.FindAll(e => e.mType == PromotionType.Point).Count();
256 float[] promotion = new float[2 * (boxCount * 2 + pointCount)];
257 float[] label = new float[boxCount * 2 + pointCount];
258 for (int i = 0; i < boxCount; i++)
259 {
260 var input = bpmos[i].GetInput();
261 for (int j = 0; j < input.Count(); j++)
262 {
263 promotion[4 * i + j] = input[j];
264 }
265 var la = bpmos[i].GetLable();
266 for (int j = 0; j < la.Count(); j++)
267 {
268 label[2 * i + j] = la[j];
269 }
270 }
271 for (int i = 0; i < pointCount; i++)
272 {
273 var p = pproms[i].GetInput();
274 for (int j = 0; j < p.Count(); j++)
275 {
276 promotion[boxCount * 4 + 2 * i + j] = p[j];
277 }
278 var la = pproms[i].GetLable();
279 for (int j = 0; j < la.Count(); j++)
280 {
281 label[boxCount * 2 + i + j] = la[j];
282 }
283 }
284
285 var point_coords_tensor = new DenseTensor<float>(promotion, new[] { 1, boxCount * 2 + pointCount, 2 });
286
287 var point_label_tensor = new DenseTensor<float>(label, new[] { 1, boxCount * 2 + pointCount });
288
289 float[] mask = new float[256 * 256];
290 for (int i = 0; i < mask.Count(); i++)
291 {
292 mask[i] = 0;
293 }
294 var mask_tensor = new DenseTensor<float>(mask, new[] { 1, 1, 256, 256 });
295
296 float[] hasMaskValues = new float[1] { 0 };
297 var hasMaskValues_tensor = new DenseTensor<float>(hasMaskValues, new[] { 1 });
298 List<NamedOnnxValue> decode_inputs;
299 if(SAM.SAM2)
300 {
301 int[] orig_im_size_values = { (int)orgHei, (int)orgWid };
302 var orig_im_size_values_tensor = new DenseTensor<int>(orig_im_size_values, new[] { 2 });
303 var fs = new float[32 * 256 * 256];
304 var feats = new DenseTensor<float>(fs, new[] { 1, 32, 256, 256 });
305 var fs2 = new float[64 * 128 * 128];
306 var feats2 = new DenseTensor<float>(fs2, new[] { 1, 64, 128, 128 });
307 decode_inputs = new List<NamedOnnxValue>
308 {
309 NamedOnnxValue.CreateFromTensor("image_embed", embedding_tensor),
310 NamedOnnxValue.CreateFromTensor("high_res_feats_0", feats),
311 NamedOnnxValue.CreateFromTensor("high_res_feats_1", feats2),
312 NamedOnnxValue.CreateFromTensor("point_coords", point_coords_tensor),
313 NamedOnnxValue.CreateFromTensor("point_labels", point_label_tensor),
314 NamedOnnxValue.CreateFromTensor("mask_input", mask_tensor),
315 NamedOnnxValue.CreateFromTensor("has_mask_input", hasMaskValues_tensor),
316 NamedOnnxValue.CreateFromTensor("orig_im_size", orig_im_size_values_tensor)
317 };
318 }
319 else
320 {
321 float[] orig_im_size_values = { (float)orgHei, (float)orgWid };
322 var orig_im_size_values_tensor = new DenseTensor<float>(orig_im_size_values, new[] { 2 });
323 decode_inputs = new List<NamedOnnxValue>
324 {
325 NamedOnnxValue.CreateFromTensor("image_embeddings", embedding_tensor),
326 NamedOnnxValue.CreateFromTensor("point_coords", point_coords_tensor),
327 NamedOnnxValue.CreateFromTensor("point_labels", point_label_tensor),
328 NamedOnnxValue.CreateFromTensor("mask_input", mask_tensor),
329 NamedOnnxValue.CreateFromTensor("has_mask_input", hasMaskValues_tensor),
330 NamedOnnxValue.CreateFromTensor("orig_im_size", orig_im_size_values_tensor)
331 };
332 }
333 MaskData md = new MaskData();
334 var segmask = this.mDecoder.Run(decode_inputs).ToList();
335 md.mMask = segmask[0].AsTensor<float>().ToArray().ToList();
336 md.mShape = segmask[0].AsTensor<float>().Dimensions.ToArray();
337 md.mIoU = segmask[1].AsTensor<float>().ToList();
338 BioLib.Recorder.AddLine("MaskData md = App.samTool.sam.Decode(App.samTool.Promotions,(float[])ImageView.SelectedImage.Tag," + orgWid + "," + orgHei, false);
339 return md;
340
341 }
342
343 public static void RemoveDuplicates(bool removeDistance, float distance, bool removeLarge, float largeArea)
344 {
345 // Track duplicates to remove
346 HashSet<ROI> dups = new HashSet<ROI>();
347
348 // Loop through all Z, C, T dimensions
349 for (int z = 0; z < ImageView.SelectedImage.SizeZ; z++)
350 {
351 for (int c = 0; c < ImageView.SelectedImage.SizeC; c++)
352 {
353 for (int t = 0; t < ImageView.SelectedImage.SizeT; t++)
354 {
355 // Collect all ROIs matching the current Z, C, T coordinates
356 List<ROI> rois = ImageView.SelectedImage.Annotations
357 .Where(annotation => annotation.coord.Equals(new ZCT(z, c, t)))
358 .ToList();
359
360 // Remove ROIs that exceed the large area threshold
361 if (removeLarge)
362 {
363 double imageArea = (ImageView.SelectedImage.SizeX) * ImageView.SelectedImage.Resolutions[0].PhysicalSizeX *
364 (ImageView.SelectedImage.SizeY) * ImageView.SelectedImage.Resolutions[0].PhysicalSizeY;
365
366 foreach (var roi in rois)
367 {
368 double roiArea = 0;
369 if(roi.type == ROI.Type.Mask)
370 {
371 roiArea = (roi.roiMask.Width * roi.roiMask.PhysicalSizeX) * (roi.roiMask.Height * roi.roiMask.PhysicalSizeY);
372 }
373 else
374 roiArea = roi.BoundingBox.W * roi.BoundingBox.H;
375 if (roiArea > largeArea * imageArea) // Check if ROI exceeds the threshold
376 {
377 dups.Add(roi);
378 }
379 }
380 }
381 double px = ImageView.SelectedImage.PhysicalSizeX;
382 double py = ImageView.SelectedImage.PhysicalSizeY;
383 // Remove ROIs that are too close to each other
384 if (removeDistance)
385 {
386 for (int i = 0; i < rois.Count - 1; i++)
387 {
388 ROI roi1 = rois[i];
389 if(dups.Contains(roi1))
390 continue;
391 for (int j = i + 1; j < rois.Count; j++)
392 {
393 ROI roi2 = rois[j];
394 if (dups.Contains(roi2))
395 continue;
396 // Calculate Euclidean distance between centers
397 double centerX1 = ImageView.SelectedImage.StageSizeX + (roi1.roiMask.X * px) + ((roi1.roiMask.Width * px) / 2);
398 double centerY1 = ImageView.SelectedImage.StageSizeY + (roi1.roiMask.Y * py) + ((roi1.roiMask.Height * py) / 2);
399 double centerX2 = ImageView.SelectedImage.StageSizeX + (roi2.roiMask.X * px) + ((roi2.roiMask.Width * px) / 2);
400 double centerY2 = ImageView.SelectedImage.StageSizeY + (roi2.roiMask.Y * py) + ((roi2.roiMask.Height * py) / 2);
401
402 double deltaX = centerX2 - centerX1;
403 double deltaY = centerY2 - centerY1;
404 double dist = Math.Sqrt(deltaX * deltaX + deltaY * deltaY);
405
406 // Mark the second ROI as duplicate if distance is below threshold
407 if (dist < distance)
408 {
409 dups.Add(roi2); // Keep roi1 by default
410 }
411 }
412 }
413 }
414
415 // Remove duplicates from the annotations list
416 foreach (var duplicate in dups)
417 {
418 ImageView.SelectedImage.Annotations.Remove(duplicate);
419 }
420
421 // Clear duplicates list for the next iteration
422 dups.Clear();
423 }
424 }
425 }
426
427 // Record the operation
428 BioLib.Recorder.Record($"SAM.RemoveDuplicates({removeDistance}, {distance}, {removeLarge}, {largeArea});");
429 }
430
431 static Random rng = new Random();
432 public static void Run(float minArea, float maxArea,float stability,float prediction, int layers, int points)
433 {
434 AutoSAM asm = new AutoSAM(points, 64, prediction, stability, 1, 0.7f, layers, 0.7f, 0.3413333f, 1, null, 0, "binary_mask");
435 for (int z = 0; z < ImageView.SelectedImage.SizeZ; z++)
436 {
437 for (int c = 0; c < ImageView.SelectedImage.SizeC; c++)
438 {
439 for (int t = 0; t < ImageView.SelectedImage.SizeT; t++)
440 {
441 MaskData md = asm.Generate(ImageView.SelectedImage, ImageView.SelectedImage.Coords[z, c, t]);
442 for (int i = 0; i < md.mfinalMask.Count; i++)
443 {
444 if (md.mfinalMask[i].Count < ImageView.SelectedImage.SizeX * ImageView.SelectedImage.SizeY)
445 {
446 md.mfinalMask[i] = null;
447 md.mfinalMask.RemoveAt(i);
448 continue;
449 }
450 PointD loc = new PointD(ImageView.SelectedImage.StageSizeX, ImageView.SelectedImage.StageSizeY);
451 ROI an = ROI.CreateMask(new ZCT(z, c, t), md.mfinalMask[i].ToArray(), ImageView.SelectedImage.SizeX, ImageView.SelectedImage.SizeY, loc, ImageView.SelectedImage.PhysicalSizeX, ImageView.SelectedImage.PhysicalSizeY);
452 if (an.W * an.H < minArea)
453 continue;
454 if (an.W * an.H > maxArea)
455 continue;
456 byte r = (byte)rng.Next(0, 255);
457 byte g = (byte)rng.Next(0, 255);
458 byte bb = (byte)rng.Next(0, 255);
459 Color fc = Color.FromArgb(r, g, bb);
460 an.fillColor = fc;
461 ImageView.SelectedImage.Annotations.Add(an);
462 }
463 md.Dispose();
464 }
465 }
466 }
467 BioLib.Recorder.AddLine("SAM.Run(SAM.Promotions," + minArea + "," + maxArea + "," + stability + "," + prediction + "," + layers + "," + points + ");",false);
468 }
469
471 public void Dispose()
472 {
473 mEncoder.Dispose();
474 mDecoder.Dispose();
475 GC.Collect();
476 }
477 }
478}
BioGTK.MaskData
A structure for storing masks and their related data in batched format. Implements basic filtering an...
Definition MaskData.cs:14
BioGTK.SAM.Decode
float[] Decode(BioImage b, List< Promotion > promotions, int orgWid, int orgHei)
Definition SAM.cs:153
BioGTK.SAM.Instance
static SAM Instance()
Definition SAM.cs:33
BioGTK.SAM.Dispose
void Dispose()
The Dispose function disposes of resources and sets variables to null.
Definition SAM.cs:471
BioGTK.SAM.Encode
void Encode(BioImage b)
Definition SAM.cs:69
BioGTK.SAM.LoadONNXModel
void LoadONNXModel()
The function LoadONNXModel loads ONNX models for encoding and decoding.
Definition SAM.cs:43
BioGTK.Transforms.ApplyImage
float[] ApplyImage(Bitmap b)
Definition Transforms.cs:29
BioGTK
Definition ML.cs:17
BioLib
Definition Zarr.cs:20