Try some more algorithms

We got the black background, receipt foreground working, but this is a limited test case. I do not have something working in the general case nor the expertise to know how to do that.

TODO.txt

@@ -1,3 +1,22 @@
+-Add ImageMagick attribution to about page
+-auto detect receipts in an image and auto-crop?
+    -https://imagemagick.org/api/feature.php#gsc.tab=0 canny edge image
+    -https://blog.jiayu.co/2019/05/edge-detection-with-imagemagick/
+    -https://pyimagesearch.com/2021/10/27/automatically-ocring-receipts-and-scans/ using open CV
+    -https://www.kaggle.com/code/dmitryyemelyanov/receipt-ocr-part-1-image-segmentation-by-opencv manip done before edge detect
+    -https://www.luisllamas.es/en/how-to-use-opencv-in-net-with-opencvsharp/ (some basic code but also line detect)
+    -opencv for macOS? https://www.nuget.org/packages/OpenCvSharp4.runtime.osx.10.15-universal
+        macOS arm64: https://www.nuget.org/packages/OpenCvSharp4.runtime.osx_arm64/4.8.1-rc
+        -can use the normal nuget for windows, linnux
+        -if we can get openCV working then we can probably hack something together...
+        -https://github.com/shimat/opencvsharp/issues/949 -- requires ffmpeg?!
+        -https://github.com/shimat/opencvsharp
+    -https://www.emgu.com/wiki/index.php?title=Main_Page (GPL...)
+    -https://stackoverflow.com/questions/30296710/detecting-paper-edge-and-crop-it
+    //https://developers.goalist.co.jp/entry/2019/02/13/150126
+
+
+---------------
 *-add more items
 *-save last opened folder to settings somewhere
 
@@ -21,4 +40,5 @@
 ---Publishing---
 *-Published app has unneeded .DSYM file (fixed via .app builder)
 *-Published app has Assets folder already copied to it; don't want that in output macOS folder but it's being copied there anyway (fixed via .app builder)
-*-macOS x64 build
+*-macOS x64 build
+

src/MayShow.csproj

@@ -47,9 +47,13 @@
       <PrivateAssets Condition="'$(Configuration)' != 'Debug'">All</PrivateAssets>
     </PackageReference>
     <PackageReference Include="PDFsharp-MigraDoc" Version="6.2.3" />
-    <PackageReference Include="Magick.NET-Q16-AnyCPU" Version="14.10.2" />
+    <PackageReference Include="Magick.NET-Q8-AnyCPU" Version="14.10.3" />
     <PackageReference Include="Deadpikle.AvaloniaProgressRing" Version="0.10.11-preview20251127001" />
     <PackageReference Include="DialogHost.Avalonia" Version="0.10.4" />
     <PackageReference Include="Xaml.Behaviors.Interactions.DragAndDrop.DataGrid" Version="11.3.9.5" />
+    <PackageReference Include="OpenCvSharp4" Version="4.13.0.20260226" />
+  </ItemGroup>
+  <ItemGroup Condition=" '$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::OSX)))'  ">
+    <PackageReference Include="OpenCvSharp4.runtime.osx.10.15-universal" Version="4.7.0.20230224" />
   </ItemGroup>
 </Project>

src/ViewModels/MainViewModel.cs

@@ -20,6 +20,10 @@ using MayShow.Helpers;
 using MayShow.Interfaces;
 using MayShow.Models;
 using MayShows.Helpers;
+using OpenCvSharp;
+using System.Reflection.Metadata.Ecma335;
+using System.Collections.Generic;
+using System.Threading;
 
 namespace MayShow.ViewModels;
 
@@ -516,6 +520,190 @@ class MainViewModel : BaseViewModel, IFontResolver, ICanCheckShutdown
         return "report_data.json";
     }
 
+    public void TestReceiptFinding(object f) => TestReceiptFindingImpl((ReportFile)f);
+
+
+    private Mat? b_algor(ReportFile file)
+    {
+        using var orig = new Mat(file.FilePath);
+        using var src = new Mat(file.FilePath, ImreadModes.Grayscale);
+        if (src.Empty())
+        {
+            LogInfo("File was empty?");
+            return null;
+        }
+        using var blur = orig.GaussianBlur(new Size(5, 5), 0.0);
+        using var kernel = Cv2.GetStructuringElement(MorphShapes.Rect, new Size(9,9));
+        using var dilated = blur.Dilate(kernel, anchor: null, iterations: 4);
+        using var edges = dilated.Canny(100, 200, 3);
+        using var heirarchy = new Mat();
+        Cv2.FindContours(edges, out Mat[] contours, heirarchy, RetrievalModes.External, ContourApproximationModes.ApproxSimple);
+        using var orgWithContours = new Mat();
+        orig.CopyTo(orgWithContours);
+        Cv2.DrawContours(orgWithContours, contours, -1, Scalar.Cyan, 3);
+        // largest contours
+        var largestContours = contours.OrderByDescending(x => x.ContourArea()).Take(1).ToArray();
+        var orgWithLargestContours = new Mat();
+        orig.CopyTo(orgWithLargestContours);
+        Cv2.DrawContours(orgWithLargestContours, largestContours, -1, Scalar.Cyan, 5);
+
+        using (new OpenCvSharp.Window("blur", blur))
+        using (new OpenCvSharp.Window("dilated", dilated))
+        using (new OpenCvSharp.Window("edges", edges))
+        using (new OpenCvSharp.Window("contours", orgWithContours))
+        // using (new OpenCvSharp.Window("w biggest contours", orgWithBiggestContours))
+        {
+            Cv2.WaitKey();
+        }
+
+        return orgWithLargestContours;
+    }
+
+    private void TestReceiptFindingImpl(ReportFile file)
+    {
+        LogInfo("Running receipt edge detection on file at path {0} with OpenCV {1}...", file.FilePath, Cv2.GetVersionString() ?? "");
+        using var orig = new Mat(file.FilePath);
+        using var src = new Mat(file.FilePath, ImreadModes.Grayscale);
+        using var dst = new Mat();
+        using var wContours = new Mat();
+        if (src.Empty())
+        {
+            LogInfo("File was empty?");
+            return;
+        }
+        var threshold = src.Threshold(90,255, ThresholdTypes.Binary);
+        
+        var blur = orig.GaussianBlur(new Size(3.0, 3.0), 0.0);
+        var kernel = Cv2.GetStructuringElement(MorphShapes.Rect, new Size(9,9));
+        var dilated = blur.Dilate(kernel, null, 1);
+        var edges = dilated.Canny(50, 200, 3);
+        //# Detect all contours in Canny-edged image
+        using var heirarchy = new Mat();
+        Cv2.FindContours(threshold, out Mat[] contours, heirarchy, RetrievalModes.External, ContourApproximationModes.ApproxSimple);
+        var orgWithContours = new Mat();
+        orig.CopyTo(orgWithContours);
+        Cv2.DrawContours(orgWithContours, contours, -1, Scalar.Cyan, 3);
+        // # find full contours
+
+        var poly = new List<Mat>();
+        foreach (var contour in contours)
+        {
+            var hull = contour.ConvexHull();
+            poly.Add(hull.ApproxPolyDP(0.01 * Cv2.ArcLength(hull, true), false));
+        }
+        Console.WriteLine("How many? {0}", poly.Count);
+        var orgWithAllPly = new Mat();
+        orig.CopyTo(orgWithAllPly);
+        Cv2.DrawContours(orgWithAllPly, poly, -1, Scalar.Red, 3);
+
+        var largestPoly = poly.OrderByDescending(x => x.ContourArea()).Take(10).ToArray();
+        var orgWithLargestContoursPly = new Mat();
+        orig.CopyTo(orgWithLargestContoursPly);
+        Cv2.DrawContours(orgWithLargestContoursPly, largestPoly, -1, Scalar.Red, 5);
+        //  using (new OpenCvSharp.Window("poly", orgWithAllPly))
+        // using (new OpenCvSharp.Window("pol2y", orgWithLargestContoursPly))
+        // {
+        //     Cv2.WaitKey();
+        // }
+        //
+
+        // # Get 10 largest contours
+        Console.WriteLine(contours);
+        var orgWithLargestContours = new Mat();
+        orig.CopyTo(orgWithLargestContours);
+        var largestContours = contours.OrderByDescending(x => x.ContourArea()).Take(10).ToArray();
+        Cv2.DrawContours(orgWithLargestContours, largestContours, -1, Scalar.Cyan, 5);
+        //
+        Mat approximate_contour(Mat contour)
+        {
+            var perimeter = Cv2.ArcLength(contour, true);
+            return contour.ApproxPolyDP(0.02 * perimeter, true);
+        }
+        Mat? get_receipt_counter(Mat[] contours)
+        {
+            var poly = new List<Mat>();
+            foreach (var contour in contours)
+            {
+                // var hull = contour.ConvexHull();
+                // var arcLength = hull.ArcLength(true);
+                // var x = contour.ApproxPolyDP(0.01 * arcLength, true);
+                // poly.Add(x);
+                
+
+
+                var approx = approximate_contour(contour);
+                if (approx.Total() == 4)
+                {
+                    return approx;
+                }
+            }
+            return null;
+        }
+
+        var highestY = 0;
+        var lowestY = 9999;
+        var highestX = 0;
+        var lowestX = 9999;
+        foreach (var contour in largestContours)
+        {
+            Console.WriteLine("Rows: {0}", contour.Rows);
+            if (contour.Rows > 10) // eliminate small things?
+            {
+                for (var i = 0; i < contour.Rows; i++)
+                {
+                    var pt = contour.At<Point>(i);
+                    if (pt.X < lowestX)
+                    {
+                        lowestX = pt.X;
+                    }
+                    if (pt.X > highestX)
+                    {
+                        highestX = pt.X;
+                    }
+                    if (pt.Y < lowestY)
+                    {
+                        lowestY = pt.Y;
+                    }
+                    if (pt.Y > highestY)
+                    {
+                        highestY = pt.Y;
+                    }                
+                }
+            }
+        }
+        Console.WriteLine("Low X: {0}, High X: {1}", lowestX, highestX);
+        Console.WriteLine("Low Y: {0}, High Y: {1}", lowestY, highestY);
+        var rect = new Rect(lowestX, lowestY, Math.Abs(highestX - lowestX), Math.Abs(highestY - lowestY));
+        Console.WriteLine(rect);
+        using var crop = new Mat(orig, rect);
+
+        using (new OpenCvSharp.Window("w largest contours", orgWithLargestContours))
+        using (new OpenCvSharp.Window("crop", crop))
+        using (new OpenCvSharp.Window("crop_b", b_algor(file)))
+        // using (new OpenCvSharp.Window("w biggest contours", orgWithBiggestContours))
+        {
+            Cv2.WaitKey();
+        }
+        return;
+
+
+        var largest = get_receipt_counter(largestContours);
+        Console.WriteLine(largest);
+        var orgWithBiggestContours = new Mat();
+        orig.CopyTo(orgWithBiggestContours);
+        Cv2.DrawContours(orgWithBiggestContours, [largest], -1, Scalar.Cyan, 3);
+        ////
+        using (new OpenCvSharp.Window("src image", src))
+        // using (new OpenCvSharp.Window("blur image", blur))
+        // using (new OpenCvSharp.Window("dilated image", dilated))
+        using (new OpenCvSharp.Window("orig with all contours", orgWithContours))
+        using (new OpenCvSharp.Window("w largest contours", orgWithLargestContours))
+        // using (new OpenCvSharp.Window("w biggest contours", orgWithBiggestContours))
+        {
+            Cv2.WaitKey();
+        }
+    }
+
     public byte[]? GetFont(string faceName)
     {
         LogInfo(string.Format("Loading font {0}", faceName));

src/Views/MainView.axaml

@@ -210,6 +210,12 @@
                                 <TextBlock FontSize="12"><Run Text="&#xf07c;" FontFamily="{StaticResource FontAwesomeSolid}"/> Open File</TextBlock>
                             </Button.Content>
                         </Button>
+                        <Button Command="{Binding $parent[DataGrid].((vm:MainViewModel)DataContext).TestReceiptFinding}" 
+                                CommandParameter="{Binding}">
+                            <Button.Content>
+                                <TextBlock FontSize="12"><Run Text="&#xf07c;" FontFamily="{StaticResource FontAwesomeSolid}"/> TEST RECEIPT FIND</TextBlock>
+                            </Button.Content>
+                        </Button>
                     </StackPanel>
                 </Grid>
             </DataTemplate>