BioGTK  6.0.0
A .NET library & program for annotating, editing various microscopy imaging formats using Bioformats supported images. Including whole slide, pyramidal, and series.
•All Classes Namespaces Functions
Public Member Functions | Static Public Member Functions | Public Attributes | Static Public Attributes | Properties | List of all members
BioGTK.MicroSAM Class Reference
Inheritance diagram for BioGTK.MicroSAM:
BioGTK.SAM

Public Member Functions

new void LoadONNXModel ()
 The function LoadONNXModel loads ONNX models for encoding and decoding.
 
new void Encode ()
 
new void Encode (BioImage b)
 
new float[] Decode (BioImage b, List< Promotion > promotions, int orgWid, int orgHei)
 
new MaskData Decode (List< Promotion > promotions, float[] embedding, int orgWid, int orgHei)
 
new void Dispose ()
 The Dispose function disposes of resources and sets variables to null.
 
- Public Member Functions inherited from BioGTK.SAM
void LoadONNXModel ()
 The function LoadONNXModel loads ONNX models for encoding and decoding.
 
void Encode ()
 
void Encode (BioImage b)
 
float[] Decode (BioImage b, List< Promotion > promotions, int orgWid, int orgHei)
 
MaskData Decode (List< Promotion > promotions, float[] embedding, int orgWid, int orgHei)
 
void Dispose ()
 The Dispose function disposes of resources and sets variables to null.
 

Static Public Member Functions

static new MicroSAM Instance ()
 
static float[] ResizeTensor (float[] tensor, int sizeX, int sizeY, int newSizeX, int newSizeY)
 Resizes a tensor of shape [1, sizeX, sizeY] using bilinear interpolation.
 
- Static Public Member Functions inherited from BioGTK.SAM
static SAM Instance ()
 
static void RemoveDuplicates (bool removeDistance, float distance, bool removeLarge, float largeArea)
 
static void Run (float minArea, float maxArea, float stability, float prediction, int layers, int points)
 

Public Attributes

float mask_threshold = 0.0f
 
- Public Attributes inherited from BioGTK.SAM
float mask_threshold = 0.0f
 

Static Public Attributes

static MicroSAM theSingleton = null
 
- Static Public Attributes inherited from BioGTK.SAM
static bool SAM2 = true
 
static SAM theSingleton = null
 

Properties

bool Initialized [get]
 

Detailed Description

Definition at line 19 of file MicroSAM.cs.

Constructor & Destructor Documentation

◆ MicroSAM()

BioGTK.MicroSAM.MicroSAM ( )
protected

Definition at line 29 of file MicroSAM.cs.

30 {
31
32 }

Member Function Documentation

◆ Decode() [1/2]

new float[] BioGTK.MicroSAM.Decode ( BioImage b,
List< Promotion > promotions,
int orgWid,
int orgHei )

The function takes a list of promotions, original width, and original height as input, and returns an array of decoded output masks.

Parameters
promotionsA list of Promotion objects. Each Promotion object has properties like mType, mInput, and mLable.
orgWidThe parameter orgWid represents the original width of the image.
orgHeiThe parameter orgHei represents the original height of the image.
Returns
The method is returning an array of floats, which represents the output mask.

Definition at line 152 of file MicroSAM.cs.

153 {
154 ZCT c = App.viewer.GetCoordinate();
155 List<float[]> lts = (List<float[]>)b.Tag;
156 int fr = b.GetFrameIndex(c.Z, c.C, c.T);
157 var embedding_tensor = new DenseTensor<float>(lts[fr], new[] { 1, 256, 64, 64 });
158 var bpmos = promotions.FindAll(e => e.mType == PromotionType.Box);
159 var pproms = promotions.FindAll(e => e.mType == PromotionType.Point);
160 int boxCount = promotions.FindAll(e => e.mType == PromotionType.Box).Count();
161 int pointCount = promotions.FindAll(e => e.mType == PromotionType.Point).Count();
162 float[] promotion = new float[2 * (boxCount * 2 + pointCount)];
163 float[] label = new float[boxCount * 2 + pointCount];
164 for (int i = 0; i < boxCount; i++)
165 {
166 var input = bpmos[i].GetInput();
167 for (int j = 0; j < input.Count(); j++)
168 {
169 promotion[4 * i + j] = input[j];
170 }
171 var la = bpmos[i].GetLable();
172 for (int j = 0; j < la.Count(); j++)
173 {
174 label[2 * i + j] = la[j];
175 }
176 }
177 for (int i = 0; i < pointCount; i++)
178 {
179 var p = pproms[i].GetInput();
180 for (int j = 0; j < p.Count(); j++)
181 {
182 promotion[boxCount * 4 + 2 * i + j] = p[j];
183 }
184 var la = pproms[i].GetLable();
185 for (int j = 0; j < la.Count(); j++)
186 {
187 label[boxCount * 2 + i + j] = la[j];
188 }
189 }
190
191 var point_coords_tensor = new DenseTensor<float>(promotion, new[] { 1, boxCount * 2 + pointCount, 2 });
192
193 var point_label_tensor = new DenseTensor<float>(label, new[] { 1, boxCount * 2 + pointCount });
194
195 float[] mask = new float[256 * 256];
196 for (int i = 0; i < mask.Count(); i++)
197 {
198 mask[i] = 0;
199 }
200 var mask_tensor = new DenseTensor<float>(mask, new[] { 1, 1, 256, 256 });
201
202 float[] hasMaskValues = new float[1] { 0 };
203 var hasMaskValues_tensor = new DenseTensor<float>(hasMaskValues, new[] { 1 });
204
205 var decode_inputs = new List<NamedOnnxValue>();
206
207 float[] orig_im_size_values = { (float)orgHei, (float)orgWid };
208 var orig_im_size_values_tensor = new DenseTensor<float>(orig_im_size_values, new[] { 2 });
209 decode_inputs = new List<NamedOnnxValue>
210 {
211 NamedOnnxValue.CreateFromTensor("image_embeddings", embedding_tensor),
212 NamedOnnxValue.CreateFromTensor("point_coords", point_coords_tensor),
213 NamedOnnxValue.CreateFromTensor("point_labels", point_label_tensor),
214 NamedOnnxValue.CreateFromTensor("mask_input", mask_tensor),
215 NamedOnnxValue.CreateFromTensor("has_mask_input", hasMaskValues_tensor),
216 NamedOnnxValue.CreateFromTensor("orig_im_size", orig_im_size_values_tensor)
217 };
218
219 var segmask = this.decoder.Run(decode_inputs);
220 var outputmask = segmask.First().AsTensor<float>().ToArray();
221 BioLib.Recorder.AddLine("App.samTool.microsam.Decode(Images.GetImage(\"" + b.Filename + "\"),App.samTool.Promotions," + orgWid + "," + orgHei + ");", false);
222 return outputmask;
223 }

◆ Decode() [2/2]

new MaskData BioGTK.MicroSAM.Decode ( List< Promotion > promotions,
float[] embedding,
int orgWid,
int orgHei )

Definition at line 225 of file MicroSAM.cs.

226 {
227 var embedding_tensor = new DenseTensor<float>(embedding, new[] { 1, 256, 64, 64 });
228
229 var bpmos = promotions.FindAll(e => e.mType == PromotionType.Box);
230 var pproms = promotions.FindAll(e => e.mType == PromotionType.Point);
231 int boxCount = promotions.FindAll(e => e.mType == PromotionType.Box).Count();
232 int pointCount = promotions.FindAll(e => e.mType == PromotionType.Point).Count();
233 float[] promotion = new float[2 * (boxCount * 2 + pointCount)];
234 float[] label = new float[boxCount * 2 + pointCount];
235 for (int i = 0; i < boxCount; i++)
236 {
237 var input = bpmos[i].GetInput();
238 for (int j = 0; j < input.Count(); j++)
239 {
240 promotion[4 * i + j] = input[j];
241 }
242 var la = bpmos[i].GetLable();
243 for (int j = 0; j < la.Count(); j++)
244 {
245 label[2 * i + j] = la[j];
246 }
247 }
248 for (int i = 0; i < pointCount; i++)
249 {
250 var p = pproms[i].GetInput();
251 for (int j = 0; j < p.Count(); j++)
252 {
253 promotion[boxCount * 4 + 2 * i + j] = p[j];
254 }
255 var la = pproms[i].GetLable();
256 for (int j = 0; j < la.Count(); j++)
257 {
258 label[boxCount * 2 + i + j] = la[j];
259 }
260 }
261
262 var point_coords_tensor = new DenseTensor<float>(promotion, new[] { 1, boxCount * 2 + pointCount, 2 });
263
264 var point_label_tensor = new DenseTensor<float>(label, new[] { 1, boxCount * 2 + pointCount });
265
266 float[] mask = new float[256 * 256];
267 for (int i = 0; i < mask.Count(); i++)
268 {
269 mask[i] = 0;
270 }
271 var mask_tensor = new DenseTensor<float>(mask, new[] { 1, 1, 256, 256 });
272
273 float[] hasMaskValues = new float[1] { 0 };
274 var hasMaskValues_tensor = new DenseTensor<float>(hasMaskValues, new[] { 1 });
275 List<NamedOnnxValue> decode_inputs;
276
277 float[] orig_im_size_values = { (float)orgHei, (float)orgWid };
278 var orig_im_size_values_tensor = new DenseTensor<float>(orig_im_size_values, new[] { 2 });
279 decode_inputs = new List<NamedOnnxValue>
280 {
281 NamedOnnxValue.CreateFromTensor("image_embeddings", embedding_tensor),
282 NamedOnnxValue.CreateFromTensor("point_coords", point_coords_tensor),
283 NamedOnnxValue.CreateFromTensor("point_labels", point_label_tensor),
284 NamedOnnxValue.CreateFromTensor("mask_input", mask_tensor),
285 NamedOnnxValue.CreateFromTensor("has_mask_input", hasMaskValues_tensor),
286 NamedOnnxValue.CreateFromTensor("orig_im_size", orig_im_size_values_tensor)
287 };
288
289 MaskData md = new MaskData();
290 var segmask = this.decoder.Run(decode_inputs).ToList();
291 md.mMask = segmask[0].AsTensor<float>().ToArray().ToList();
292 md.mShape = segmask[0].AsTensor<float>().Dimensions.ToArray();
293 md.mIoU = segmask[1].AsTensor<float>().ToList();
294 BioLib.Recorder.AddLine("MaskData md = App.samTool.microsam.Decode(App.samTool.Promotions,(float[])ImageView.SelectedImage.Tag," + orgWid + "," + orgHei + ");", false);
295 return md;
296
297 }
BioGTK.MaskData
A structure for storing masks and their related data in batched format. Implements basic filtering an...
Definition MaskData.cs:14

◆ Dispose()

new void BioGTK.MicroSAM.Dispose ( )

The Dispose function disposes of resources and sets variables to null.

Definition at line 353 of file MicroSAM.cs.

354 {
355 encoder.Dispose();
356 decoder.Dispose();
357 GC.Collect();
358 }

◆ Encode() [1/2]

new void BioGTK.MicroSAM.Encode ( )

Definition at line 59 of file MicroSAM.cs.

60 {
61 Encode(ImageView.SelectedImage);
62 }

◆ Encode() [2/2]

new void BioGTK.MicroSAM.Encode ( BioImage b)

The Encode function takes a BioImage object, applies a transformation to it, converts it to a tensor, runs it through an encoder model, and stores the resulting embedding.

Parameters
BioImageThe BioImage parameter is an object that represents an image. It likely contains information such as the image data, size, and other properties related to the image.

Definition at line 69 of file MicroSAM.cs.

70 {
71 if (b.Buffers.Count * 3 * 1024 * 1024 > 4e8)
72 {
73 // Create the message dialog
74 MessageDialog msgBox = new MessageDialog(
75 null,
76 DialogFlags.Modal,
77 MessageType.Info,
78 ButtonsType.Ok | ButtonsType.Cancel,
79 "Memory required is more than 4GB are you sure you want to continue?"
80 );
81
82 // Show the message dialog
83 if (msgBox.Run() != (int)ResponseType.Ok)
84 return;
85 }
86 Progress pr = Progress.Create("Encoding Image", "Transforming", "");
87 // update ui on main UI thread
88 Application.Invoke(delegate
89 {
90 pr.Show();
91 pr.Present();
92 });
93 int i = 0;
94 foreach (Bitmap bu in b.Buffers)
95 {
96 Transforms tranform = new Transforms(1024);
97 float[] img = tranform.ApplyImage(bu);
98 var tensor = new DenseTensor<float>(img, new[] { 1, 3, 1024, 1024 });
99 IDisposableReadOnlyCollection<DisposableNamedOnnxValue> results = null;
100 List<NamedOnnxValue> inputs = new List<NamedOnnxValue>
101 {
102 NamedOnnxValue.CreateFromTensor("image", tensor)
103 };
104 try
105 {
106 results = this.encoder.Run(inputs);
107 }
108 catch (Exception e)
109 {
110 Console.WriteLine("Tried running SAM-2 next trying SAM-1");
111 SAM2 = false;
112 }
113 if (!SAM2)
114 {
115 inputs = new List<NamedOnnxValue>
116 {
117 NamedOnnxValue.CreateFromTensor("x", tensor)
118 };
119 results = this.encoder.Run(inputs);
120 }
121 if (b.Tag == null)
122 {
123 b.Tag = new List<float[]>();
124 }
125 List<float[]> l = (List<float[]>)b.Tag;
126 if (!SAM2)
127 l.Add(results.First().AsTensor<float>().ToArray());
128 else
129 l.Add(results.Last().AsTensor<float>().ToArray());
130 b.Tag = l;
131 results.Dispose();
132 pr.ProgressValue = (double)i / b.Buffers.Count;
133 i++;
134 }
135 // update ui on main UI thread
136 Application.Invoke(delegate
137 {
138 pr.Hide();
139 });
140 BioLib.Recorder.AddLine("MicroSAM.Encode(Images.GetImage(\"" + b.Filename + "\"));", false);
141 }
BioGTK.Transforms.ApplyImage
float[] ApplyImage(Bitmap b)
Definition Transforms.cs:29

◆ Instance()

static new MicroSAM BioGTK.MicroSAM.Instance ( )
static

The function returns an instance of the SAM class, creating it if it doesn't already exist.

Returns
The method is returning an instance of the SAM class.

Definition at line 36 of file MicroSAM.cs.

37 {
38 if (null == theSingleton)
39 {
40 theSingleton = new MicroSAM();
41 }
42 return theSingleton;
43 }

◆ LoadONNXModel()

new void BioGTK.MicroSAM.LoadONNXModel ( )

The function LoadONNXModel loads ONNX models for encoding and decoding.

Definition at line 46 of file MicroSAM.cs.

47 {
48 if (this.decoder != null)
49 this.decoder.Dispose();
50 if (this.encoder != null)
51 this.encoder.Dispose();
52 string exePath = Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
53 string decoder_model_path = exePath + "/micro-sam-decoder.onnx";
54 this.decoder = new InferenceSession(decoder_model_path);
55 string encoder_model_path = exePath + "/micro-sam-encoder.onnx";
56 this.encoder = new InferenceSession(encoder_model_path);
57
58 }

◆ ResizeTensor()

static float[] BioGTK.MicroSAM.ResizeTensor ( float[] tensor,
int sizeX,
int sizeY,
int newSizeX,
int newSizeY )
static

Resizes a tensor of shape [1, sizeX, sizeY] using bilinear interpolation.

Parameters
tensorThe input tensor as a flat array.
sizeXThe width of the input tensor.
sizeYThe height of the input tensor.
newSizeXThe desired width of the output tensor.
newSizeYThe desired height of the output tensor.
Returns
A resized tensor as a flat array.

Definition at line 307 of file MicroSAM.cs.

308 {
309 float[] resizedTensor = new float[newSizeX * newSizeY];
310
311 // Scale factors for mapping output to input
312 float scaleX = (float)sizeX / newSizeX;
313 float scaleY = (float)sizeY / newSizeY;
314
315 for (int y = 0; y < newSizeY; y++)
316 {
317 for (int x = 0; x < newSizeX; x++)
318 {
319 // Map the new coordinates to the original
320 float origX = x * scaleX;
321 float origY = y * scaleY;
322
323 // Get the integer and fractional parts
324 int x0 = (int)Math.Floor(origX);
325 int y0 = (int)Math.Floor(origY);
326 int x1 = Math.Min(x0 + 1, sizeX - 1);
327 int y1 = Math.Min(y0 + 1, sizeY - 1);
328
329 float xFrac = origX - x0;
330 float yFrac = origY - y0;
331
332 // Get the values at the four surrounding points
333 float topLeft = tensor[y0 * sizeX + x0];
334 float topRight = tensor[y0 * sizeX + x1];
335 float bottomLeft = tensor[y1 * sizeX + x0];
336 float bottomRight = tensor[y1 * sizeX + x1];
337
338 // Perform bilinear interpolation
339 float top = topLeft + xFrac * (topRight - topLeft);
340 float bottom = bottomLeft + xFrac * (bottomRight - bottomLeft);
341 float value = top + yFrac * (bottom - top);
342
343 // Assign the computed value to the resized tensor
344 resizedTensor[y * newSizeX + x] = value;
345 }
346 }
347
348 return resizedTensor;
349 }

Member Data Documentation

◆ mask_threshold

float BioGTK.MicroSAM.mask_threshold = 0.0f

Definition at line 24 of file MicroSAM.cs.

◆ theSingleton

MicroSAM BioGTK.MicroSAM.theSingleton = null
static

Definition at line 21 of file MicroSAM.cs.

Property Documentation

◆ Initialized

bool BioGTK.MicroSAM.Initialized
get

Definition at line 25 of file MicroSAM.cs.

26 {
27 get { if (encoder != null) return true; else return false; }
28 }
The documentation for this class was generated from the following file: