BioGTK  6.0.0
A .NET library & program for annotating, editing various microscopy imaging formats using Bioformats supported images. Including whole slide, pyramidal, and series.
•All Classes Namespaces Functions
Public Member Functions | Static Public Member Functions | Public Attributes | Static Public Attributes | List of all members
BioGTK.SAM Class Reference
Inheritance diagram for BioGTK.SAM:
BioGTK.MicroSAM

Public Member Functions

void LoadONNXModel ()
 The function LoadONNXModel loads ONNX models for encoding and decoding.
 
void Encode ()
 
void Encode (BioImage b)
 
float[] Decode (BioImage b, List< Promotion > promotions, int orgWid, int orgHei)
 
MaskData Decode (List< Promotion > promotions, float[] embedding, int orgWid, int orgHei)
 
void Dispose ()
 The Dispose function disposes of resources and sets variables to null.
 

Static Public Member Functions

static SAM Instance ()
 
static void RemoveDuplicates (bool removeDistance, float distance, bool removeLarge, float largeArea)
 
static void Run (float minArea, float maxArea, float stability, float prediction, int layers, int points)
 

Public Attributes

float mask_threshold = 0.0f
 

Static Public Attributes

static bool SAM2 = true
 
static SAM theSingleton = null
 

Detailed Description

Definition at line 19 of file SAM.cs.

Constructor & Destructor Documentation

◆ SAM()

BioGTK.SAM.SAM ( )
protected

Definition at line 27 of file SAM.cs.

28 {
29 }

Member Function Documentation

◆ Decode() [1/2]

float[] BioGTK.SAM.Decode ( BioImage b,
List< Promotion > promotions,
int orgWid,
int orgHei )

The function takes a list of promotions, original width, and original height as input, and returns an array of decoded output masks.

Parameters
promotionsA list of Promotion objects. Each Promotion object has properties like mType, mInput, and mLable.
orgWidThe parameter orgWid represents the original width of the image.
orgHeiThe parameter orgHei represents the original height of the image.
Returns
The method is returning an array of floats, which represents the output mask.

Definition at line 153 of file SAM.cs.

154 {
155 ZCT c = App.viewer.GetCoordinate();
156 List<float[]> lts = (List<float[]>)b.Tag;
157 int fr = b.GetFrameIndex(c.Z, c.C, c.T);
158 var embedding_tensor = new DenseTensor<float>(lts[fr], new[] { 1, 256, 64, 64 });
159 var bpmos = promotions.FindAll(e => e.mType == PromotionType.Box);
160 var pproms = promotions.FindAll(e => e.mType == PromotionType.Point);
161 int boxCount = promotions.FindAll(e => e.mType == PromotionType.Box).Count();
162 int pointCount = promotions.FindAll(e => e.mType == PromotionType.Point).Count();
163 float[] promotion = new float[2 * (boxCount * 2 + pointCount)];
164 float[] label = new float[boxCount * 2 + pointCount];
165 for (int i = 0; i < boxCount; i++)
166 {
167 var input = bpmos[i].GetInput();
168 for (int j = 0; j < input.Count(); j++)
169 {
170 promotion[4 * i + j] = input[j];
171 }
172 var la = bpmos[i].GetLable();
173 for (int j = 0; j < la.Count(); j++)
174 {
175 label[2 * i + j] = la[j];
176 }
177 }
178 for (int i = 0; i < pointCount; i++)
179 {
180 var p = pproms[i].GetInput();
181 for (int j = 0; j < p.Count(); j++)
182 {
183 promotion[boxCount * 4 + 2 * i + j] = p[j];
184 }
185 var la = pproms[i].GetLable();
186 for (int j = 0; j < la.Count(); j++)
187 {
188 label[boxCount * 2 + i + j] = la[j];
189 }
190 }
191
192 var point_coords_tensor = new DenseTensor<float>(promotion, new[] { 1, boxCount * 2 + pointCount, 2 });
193
194 var point_label_tensor = new DenseTensor<float>(label, new[] { 1, boxCount * 2 + pointCount });
195
196 float[] mask = new float[256 * 256];
197 for (int i = 0; i < mask.Count(); i++)
198 {
199 mask[i] = 0;
200 }
201 var mask_tensor = new DenseTensor<float>(mask, new[] { 1, 1, 256, 256 });
202
203 float[] hasMaskValues = new float[1] { 0 };
204 var hasMaskValues_tensor = new DenseTensor<float>(hasMaskValues, new[] { 1 });
205
206 var decode_inputs = new List<NamedOnnxValue>();
207 if (SAM2)
208 {
209 int[] orig_im_size_values = { (int)orgHei, (int)orgWid };
210 var orig_im_size_values_tensor = new DenseTensor<int>(orig_im_size_values, new[] { 2 });
211 var fs = new float[32 * 256 * 256];
212 var feats = new DenseTensor<float>(fs, new[] { 1, 32, 256, 256 });
213 var fs2 = new float[64 * 128 * 128];
214 var feats2 = new DenseTensor<float>(fs2, new[] { 1, 64, 128, 128 });
215 decode_inputs = new List<NamedOnnxValue>
216 {
217 NamedOnnxValue.CreateFromTensor("image_embed", embedding_tensor),
218 NamedOnnxValue.CreateFromTensor("high_res_feats_0", feats),
219 NamedOnnxValue.CreateFromTensor("high_res_feats_1", feats2),
220 NamedOnnxValue.CreateFromTensor("point_coords", point_coords_tensor),
221 NamedOnnxValue.CreateFromTensor("point_labels", point_label_tensor),
222 NamedOnnxValue.CreateFromTensor("mask_input", mask_tensor),
223 NamedOnnxValue.CreateFromTensor("has_mask_input", hasMaskValues_tensor),
224 NamedOnnxValue.CreateFromTensor("orig_im_size", orig_im_size_values_tensor)
225 };
226 }
227 else
228 {
229 float[] orig_im_size_values = { (float)orgHei, (float)orgWid };
230 var orig_im_size_values_tensor = new DenseTensor<float>(orig_im_size_values, new[] { 2 });
231 decode_inputs = new List<NamedOnnxValue>
232 {
233 NamedOnnxValue.CreateFromTensor("image_embeddings", embedding_tensor),
234 NamedOnnxValue.CreateFromTensor("point_coords", point_coords_tensor),
235 NamedOnnxValue.CreateFromTensor("point_labels", point_label_tensor),
236 NamedOnnxValue.CreateFromTensor("mask_input", mask_tensor),
237 NamedOnnxValue.CreateFromTensor("has_mask_input", hasMaskValues_tensor),
238 NamedOnnxValue.CreateFromTensor("orig_im_size", orig_im_size_values_tensor)
239 };
240 }
241 var segmask = this.mDecoder.Run(decode_inputs);
242 var outputmask = segmask.First().AsTensor<float>().ToArray();
243 BioLib.Recorder.AddLine("App.samTool.sam.Decode(Images.GetImage(\"" + b.Filename + "\"),App.samTool.Promotions," + orgWid + "," + orgHei + ");", false);
244 return outputmask;
245
246 }

◆ Decode() [2/2]

MaskData BioGTK.SAM.Decode ( List< Promotion > promotions,
float[] embedding,
int orgWid,
int orgHei )

Definition at line 248 of file SAM.cs.

249 {
250 var embedding_tensor = new DenseTensor<float>(embedding, new[] { 1, 256, 64, 64 });
251
252 var bpmos = promotions.FindAll(e => e.mType == PromotionType.Box);
253 var pproms = promotions.FindAll(e => e.mType == PromotionType.Point);
254 int boxCount = promotions.FindAll(e => e.mType == PromotionType.Box).Count();
255 int pointCount = promotions.FindAll(e => e.mType == PromotionType.Point).Count();
256 float[] promotion = new float[2 * (boxCount * 2 + pointCount)];
257 float[] label = new float[boxCount * 2 + pointCount];
258 for (int i = 0; i < boxCount; i++)
259 {
260 var input = bpmos[i].GetInput();
261 for (int j = 0; j < input.Count(); j++)
262 {
263 promotion[4 * i + j] = input[j];
264 }
265 var la = bpmos[i].GetLable();
266 for (int j = 0; j < la.Count(); j++)
267 {
268 label[2 * i + j] = la[j];
269 }
270 }
271 for (int i = 0; i < pointCount; i++)
272 {
273 var p = pproms[i].GetInput();
274 for (int j = 0; j < p.Count(); j++)
275 {
276 promotion[boxCount * 4 + 2 * i + j] = p[j];
277 }
278 var la = pproms[i].GetLable();
279 for (int j = 0; j < la.Count(); j++)
280 {
281 label[boxCount * 2 + i + j] = la[j];
282 }
283 }
284
285 var point_coords_tensor = new DenseTensor<float>(promotion, new[] { 1, boxCount * 2 + pointCount, 2 });
286
287 var point_label_tensor = new DenseTensor<float>(label, new[] { 1, boxCount * 2 + pointCount });
288
289 float[] mask = new float[256 * 256];
290 for (int i = 0; i < mask.Count(); i++)
291 {
292 mask[i] = 0;
293 }
294 var mask_tensor = new DenseTensor<float>(mask, new[] { 1, 1, 256, 256 });
295
296 float[] hasMaskValues = new float[1] { 0 };
297 var hasMaskValues_tensor = new DenseTensor<float>(hasMaskValues, new[] { 1 });
298 List<NamedOnnxValue> decode_inputs;
299 if(SAM.SAM2)
300 {
301 int[] orig_im_size_values = { (int)orgHei, (int)orgWid };
302 var orig_im_size_values_tensor = new DenseTensor<int>(orig_im_size_values, new[] { 2 });
303 var fs = new float[32 * 256 * 256];
304 var feats = new DenseTensor<float>(fs, new[] { 1, 32, 256, 256 });
305 var fs2 = new float[64 * 128 * 128];
306 var feats2 = new DenseTensor<float>(fs2, new[] { 1, 64, 128, 128 });
307 decode_inputs = new List<NamedOnnxValue>
308 {
309 NamedOnnxValue.CreateFromTensor("image_embed", embedding_tensor),
310 NamedOnnxValue.CreateFromTensor("high_res_feats_0", feats),
311 NamedOnnxValue.CreateFromTensor("high_res_feats_1", feats2),
312 NamedOnnxValue.CreateFromTensor("point_coords", point_coords_tensor),
313 NamedOnnxValue.CreateFromTensor("point_labels", point_label_tensor),
314 NamedOnnxValue.CreateFromTensor("mask_input", mask_tensor),
315 NamedOnnxValue.CreateFromTensor("has_mask_input", hasMaskValues_tensor),
316 NamedOnnxValue.CreateFromTensor("orig_im_size", orig_im_size_values_tensor)
317 };
318 }
319 else
320 {
321 float[] orig_im_size_values = { (float)orgHei, (float)orgWid };
322 var orig_im_size_values_tensor = new DenseTensor<float>(orig_im_size_values, new[] { 2 });
323 decode_inputs = new List<NamedOnnxValue>
324 {
325 NamedOnnxValue.CreateFromTensor("image_embeddings", embedding_tensor),
326 NamedOnnxValue.CreateFromTensor("point_coords", point_coords_tensor),
327 NamedOnnxValue.CreateFromTensor("point_labels", point_label_tensor),
328 NamedOnnxValue.CreateFromTensor("mask_input", mask_tensor),
329 NamedOnnxValue.CreateFromTensor("has_mask_input", hasMaskValues_tensor),
330 NamedOnnxValue.CreateFromTensor("orig_im_size", orig_im_size_values_tensor)
331 };
332 }
333 MaskData md = new MaskData();
334 var segmask = this.mDecoder.Run(decode_inputs).ToList();
335 md.mMask = segmask[0].AsTensor<float>().ToArray().ToList();
336 md.mShape = segmask[0].AsTensor<float>().Dimensions.ToArray();
337 md.mIoU = segmask[1].AsTensor<float>().ToList();
338 BioLib.Recorder.AddLine("MaskData md = App.samTool.sam.Decode(App.samTool.Promotions,(float[])ImageView.SelectedImage.Tag," + orgWid + "," + orgHei, false);
339 return md;
340
341 }
BioGTK.MaskData
A structure for storing masks and their related data in batched format. Implements basic filtering an...
Definition MaskData.cs:14

◆ Dispose()

void BioGTK.SAM.Dispose ( )

The Dispose function disposes of resources and sets variables to null.

Definition at line 472 of file SAM.cs.

473 {
474 mEncoder.Dispose();
475 mDecoder.Dispose();
476 GC.Collect();
477 }

◆ Encode() [1/2]

void BioGTK.SAM.Encode ( )

Definition at line 59 of file SAM.cs.

60 {
61 Encode(ImageView.SelectedImage);
62 }

◆ Encode() [2/2]

void BioGTK.SAM.Encode ( BioImage b)

The Encode function takes a BioImage object, applies a transformation to it, converts it to a tensor, runs it through an encoder model, and stores the resulting embedding.

Parameters
BioImageThe BioImage parameter is an object that represents an image. It likely contains information such as the image data, size, and other properties related to the image.

Definition at line 69 of file SAM.cs.

70 {
71 if (b.Buffers.Count * 3 * 1024 * 1024 > 4e8)
72 {
73 // Create the message dialog
74 MessageDialog msgBox = new MessageDialog(
75 null,
76 DialogFlags.Modal,
77 MessageType.Info,
78 ButtonsType.Ok | ButtonsType.Cancel,
79 "Memory required is more than 4GB are you sure you want to continue?"
80 );
81
82 // Show the message dialog
83 if (msgBox.Run() != (int)ResponseType.Ok)
84 return;
85 }
86 Progress pr = Progress.Create("Encoding Image", "Transforming", "");
87 // update ui on main UI thread
88 Application.Invoke(delegate
89 {
90 pr.Show();
91 pr.Present();
92 });
93 int i = 0;
94 foreach (Bitmap bu in b.Buffers)
95 {
96 Transforms tranform = new Transforms(1024);
97 float[] img = tranform.ApplyImage(bu);
98 var tensor = new DenseTensor<float>(img, new[] { 1, 3, 1024, 1024 });
99 IDisposableReadOnlyCollection<DisposableNamedOnnxValue> results = null;
100 List<NamedOnnxValue> inputs = new List<NamedOnnxValue>
101 {
102 NamedOnnxValue.CreateFromTensor("image", tensor)
103 };
104 try
105 {
106 results = this.mEncoder.Run(inputs);
107 }
108 catch (Exception e)
109 {
110 Console.WriteLine("Tried running SAM-2 next trying SAM-1");
111 SAM2 = false;
112 }
113 if(!SAM2)
114 {
115 inputs = new List<NamedOnnxValue>
116 {
117 NamedOnnxValue.CreateFromTensor("x", tensor)
118 };
119 results = this.mEncoder.Run(inputs);
120 }
121 if (b.Tag == null)
122 {
123 b.Tag = new List<float[]>();
124 }
125 List<float[]> l = (List<float[]>)b.Tag;
126 if(!SAM2)
127 l.Add(results.First().AsTensor<float>().ToArray());
128 else
129 l.Add(results.Last().AsTensor<float>().ToArray());
130 b.Tag = l;
131 results.Dispose();
132 pr.ProgressValue = (double)i/b.Buffers.Count;
133 i++;
134 }
135 // update ui on main UI thread
136 Application.Invoke(delegate
137 {
138 pr.Hide();
139 });
140 this.mReady = true;
141 BioLib.Recorder.AddLine("SAM.Encode(Images.GetImage(\"" + b.Filename + "\"));", false);
142 }
BioGTK.Transforms.ApplyImage
float[] ApplyImage(Bitmap b)
Definition Transforms.cs:29

◆ Instance()

static SAM BioGTK.SAM.Instance ( )
static

The function returns an instance of the SAM class, creating it if it doesn't already exist.

Returns
The method is returning an instance of the SAM class.

Definition at line 33 of file SAM.cs.

34 {
35 if (null == theSingleton)
36 {
37 theSingleton = new SAM();
38 }
39 return theSingleton;
40 }

◆ LoadONNXModel()

void BioGTK.SAM.LoadONNXModel ( )

The function LoadONNXModel loads ONNX models for encoding and decoding.

Definition at line 43 of file SAM.cs.

44 {
45 if (this.mEncoder != null)
46 this.mEncoder.Dispose();
47
48 if (this.mDecoder != null)
49 this.mDecoder.Dispose();
50
51 string exePath = Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
52 string encode_model_path = exePath + "/encoder-quant.onnx";
53 this.mEncoder = new InferenceSession(encode_model_path);
54
55 string decode_model_path = exePath + "/decoder-quant.onnx";
56 this.mDecoder = new InferenceSession(decode_model_path);
57 SAMTool.Encode();
58 }

◆ RemoveDuplicates()

static void BioGTK.SAM.RemoveDuplicates ( bool removeDistance,
float distance,
bool removeLarge,
float largeArea )
static

Definition at line 343 of file SAM.cs.

344 {
345 // Track duplicates to remove
346 HashSet<ROI> dups = new HashSet<ROI>();
347
348 // Loop through all Z, C, T dimensions
349 for (int z = 0; z < ImageView.SelectedImage.SizeZ; z++)
350 {
351 for (int c = 0; c < ImageView.SelectedImage.SizeC; c++)
352 {
353 for (int t = 0; t < ImageView.SelectedImage.SizeT; t++)
354 {
355 // Collect all ROIs matching the current Z, C, T coordinates
356 List<ROI> rois = ImageView.SelectedImage.Annotations
357 .Where(annotation => annotation.coord.Equals(new ZCT(z, c, t)))
358 .ToList();
359
360 // Remove ROIs that exceed the large area threshold
361 if (removeLarge)
362 {
363 double imageArea = (ImageView.SelectedImage.SizeX) * ImageView.SelectedImage.Resolutions[0].PhysicalSizeX *
364 (ImageView.SelectedImage.SizeY) * ImageView.SelectedImage.Resolutions[0].PhysicalSizeY;
365
366 foreach (var roi in rois)
367 {
368 double roiArea = 0;
369 if(roi.type == ROI.Type.Mask)
370 {
371 roiArea = (roi.roiMask.Width * roi.roiMask.PhysicalSizeX) * (roi.roiMask.Height * roi.roiMask.PhysicalSizeY);
372 }
373 else
374 roiArea = roi.BoundingBox.W * roi.BoundingBox.H;
375 if (roiArea > largeArea * imageArea) // Check if ROI exceeds the threshold
376 {
377 dups.Add(roi);
378 }
379 }
380 }
381 double px = ImageView.SelectedImage.PhysicalSizeX;
382 double py = ImageView.SelectedImage.PhysicalSizeY;
383 // Remove ROIs that are too close to each other
384 if (removeDistance)
385 {
386 for (int i = 0; i < rois.Count - 1; i++)
387 {
388 ROI roi1 = rois[i];
389 if(dups.Contains(roi1))
390 continue;
391 for (int j = i + 1; j < rois.Count; j++)
392 {
393 ROI roi2 = rois[j];
394 if (dups.Contains(roi2))
395 continue;
396 // Calculate Euclidean distance between centers
397 double centerX1 = ImageView.SelectedImage.StageSizeX + (roi1.roiMask.X * px) + ((roi1.roiMask.Width * px) / 2);
398 double centerY1 = ImageView.SelectedImage.StageSizeY + (roi1.roiMask.Y * py) + ((roi1.roiMask.Height * py) / 2);
399 double centerX2 = ImageView.SelectedImage.StageSizeX + (roi2.roiMask.X * px) + ((roi2.roiMask.Width * px) / 2);
400 double centerY2 = ImageView.SelectedImage.StageSizeY + (roi2.roiMask.Y * py) + ((roi2.roiMask.Height * py) / 2);
401
402 double deltaX = centerX2 - centerX1;
403 double deltaY = centerY2 - centerY1;
404 double dist = Math.Sqrt(deltaX * deltaX + deltaY * deltaY);
405
406 // Mark the second ROI as duplicate if distance is below threshold
407 if (dist < distance)
408 {
409 dups.Add(roi2); // Keep roi1 by default
410 }
411 }
412 }
413 }
414
415 // Remove duplicates from the annotations list
416 foreach (var duplicate in dups)
417 {
418 ImageView.SelectedImage.Annotations.Remove(duplicate);
419 }
420
421 // Clear duplicates list for the next iteration
422 dups.Clear();
423 }
424 }
425 }
426
427 // Record the operation
428 BioLib.Recorder.Record($"SAM.RemoveDuplicates({removeDistance}, {distance}, {removeLarge}, {largeArea});");
429 }

◆ Run()

static void BioGTK.SAM.Run ( float minArea,
float maxArea,
float stability,
float prediction,
int layers,
int points )
static

Definition at line 432 of file SAM.cs.

433 {
434 AutoSAM asm = new AutoSAM(points, 64, prediction, stability, 1, 0.7f, layers, 0.7f, 0.3413333f, 1, null, 0, "binary_mask");
435 for (int z = 0; z < ImageView.SelectedImage.SizeZ; z++)
436 {
437 for (int c = 0; c < ImageView.SelectedImage.SizeC; c++)
438 {
439 for (int t = 0; t < ImageView.SelectedImage.SizeT; t++)
440 {
441 MaskData md = asm.Generate(ImageView.SelectedImage, ImageView.SelectedImage.Coords[z, c, t]);
442 for (int i = 0; i < md.mfinalMask.Count; i++)
443 {
444 if (md.mfinalMask[i].Count < ImageView.SelectedImage.SizeX * ImageView.SelectedImage.SizeY)
445 {
446 md.mfinalMask[i] = null;
447 md.mfinalMask.RemoveAt(i);
448 continue;
449 }
450 PointD loc = new PointD(ImageView.SelectedImage.StageSizeX, ImageView.SelectedImage.StageSizeY);
451 ROI an = ROI.CreateMask(new ZCT(z, c, t), md.mfinalMask[i].ToArray(), ImageView.SelectedImage.SizeX, ImageView.SelectedImage.SizeY, loc, ImageView.SelectedImage.PhysicalSizeX, ImageView.SelectedImage.PhysicalSizeY);
452 if (an.W * an.H < minArea)
453 continue;
454 if (an.W * an.H > maxArea)
455 continue;
456 byte r = (byte)rng.Next(0, 255);
457 byte g = (byte)rng.Next(0, 255);
458 byte bb = (byte)rng.Next(0, 255);
459 an.fillColor.R = r;
460 an.fillColor.G = g;
461 an.fillColor.B = bb;
462 ImageView.SelectedImage.Annotations.Add(an);
463 }
464 md.Dispose();
465 }
466 }
467 }
468 BioLib.Recorder.AddLine("SAM.Run(SAM.Promotions," + minArea + "," + maxArea + "," + stability + "," + prediction + "," + layers + "," + points + ");",false);
469 }

Member Data Documentation

◆ mask_threshold

float BioGTK.SAM.mask_threshold = 0.0f

Definition at line 26 of file SAM.cs.

◆ SAM2

bool BioGTK.SAM.SAM2 = true
static

Definition at line 21 of file SAM.cs.

◆ theSingleton

SAM BioGTK.SAM.theSingleton = null
static

Definition at line 22 of file SAM.cs.

The documentation for this class was generated from the following file: