diff --git a/ImageProc.cs b/ImageProc.cs index 30f5f87..f424147 100644 --- a/ImageProc.cs +++ b/ImageProc.cs @@ -19,6 +19,23 @@ internal static Bitmap CaptureScreenshot(Rectangle bounds) return bitmap; } + + internal static Bitmap CropImage(Image original, Rectangle bounds) + { + Bitmap bitmap = new Bitmap(bounds.Width + 200, bounds.Height + 200); + + using (Graphics g = Graphics.FromImage(bitmap)) + { + g.DrawImage(original, + 10, 10, + new Rectangle(bounds.Left, bounds.Top, bounds.Width, bounds.Height), + GraphicsUnit.Pixel + ); + } + + return bitmap; + } + internal static Color GetColorFromCurrentPixel() { Bitmap bitmap = new Bitmap(10, 10); diff --git a/MainForm.Designer.cs b/MainForm.Designer.cs index 1c93789..f5d9022 100644 --- a/MainForm.Designer.cs +++ b/MainForm.Designer.cs @@ -64,10 +64,14 @@ private void InitializeComponent() this.settingsLabel = new System.Windows.Forms.Label(); this.voiceComboLabel = new System.Windows.Forms.Label(); this.voiceCombo = new System.Windows.Forms.ComboBox(); + this.defaultdpiLabel = new System.Windows.Forms.Label(); + this.label2 = new System.Windows.Forms.Label(); + this.defaultdpiBar = new System.Windows.Forms.TrackBar(); ((System.ComponentModel.ISupportInitialize)(this.rawImage)).BeginInit(); ((System.ComponentModel.ISupportInitialize)(this.processedImage)).BeginInit(); ((System.ComponentModel.ISupportInitialize)(this.distanceBar)).BeginInit(); this.contextMenu.SuspendLayout(); + ((System.ComponentModel.ISupportInitialize)(this.defaultdpiBar)).BeginInit(); this.SuspendLayout(); // // ocrButton @@ -188,7 +192,7 @@ private void InitializeComponent() // distanceBar // this.distanceBar.Location = new System.Drawing.Point(740, 137); - this.distanceBar.Maximum = 100; + this.distanceBar.Maximum = 255; this.distanceBar.Name = "distanceBar"; this.distanceBar.Size = new System.Drawing.Size(139, 45); this.distanceBar.TabIndex = 12; @@ -215,7 +219,7 @@ private void InitializeComponent() // distanceLabel // this.distanceLabel.AutoSize = true; - this.distanceLabel.Location = new System.Drawing.Point(885, 142); + this.distanceLabel.Location = new System.Drawing.Point(877, 142); this.distanceLabel.Name = "distanceLabel"; this.distanceLabel.Size = new System.Drawing.Size(13, 13); this.distanceLabel.TabIndex = 15; @@ -338,7 +342,7 @@ private void InitializeComponent() // voiceComboLabel // this.voiceComboLabel.AutoSize = true; - this.voiceComboLabel.Location = new System.Drawing.Point(737, 185); + this.voiceComboLabel.Location = new System.Drawing.Point(737, 230); this.voiceComboLabel.Name = "voiceComboLabel"; this.voiceComboLabel.Size = new System.Drawing.Size(37, 13); this.voiceComboLabel.TabIndex = 19; @@ -347,17 +351,49 @@ private void InitializeComponent() // voiceCombo // this.voiceCombo.FormattingEnabled = true; - this.voiceCombo.Location = new System.Drawing.Point(740, 202); + this.voiceCombo.Location = new System.Drawing.Point(740, 247); this.voiceCombo.Name = "voiceCombo"; this.voiceCombo.Size = new System.Drawing.Size(139, 21); this.voiceCombo.TabIndex = 20; // + // defaultdpiLabel + // + this.defaultdpiLabel.AutoSize = true; + this.defaultdpiLabel.Location = new System.Drawing.Point(877, 193); + this.defaultdpiLabel.Name = "defaultdpiLabel"; + this.defaultdpiLabel.Size = new System.Drawing.Size(13, 13); + this.defaultdpiLabel.TabIndex = 23; + this.defaultdpiLabel.Text = "0"; + // + // label2 + // + this.label2.AutoSize = true; + this.label2.Location = new System.Drawing.Point(737, 172); + this.label2.Name = "label2"; + this.label2.Size = new System.Drawing.Size(62, 13); + this.label2.TabIndex = 22; + this.label2.Text = "Default DPI"; + // + // defaultdpiBar + // + this.defaultdpiBar.Location = new System.Drawing.Point(740, 188); + this.defaultdpiBar.Maximum = 600; + this.defaultdpiBar.Minimum = 50; + this.defaultdpiBar.Name = "defaultdpiBar"; + this.defaultdpiBar.Size = new System.Drawing.Size(139, 45); + this.defaultdpiBar.TabIndex = 21; + this.defaultdpiBar.Value = 50; + this.defaultdpiBar.Scroll += new System.EventHandler(this.defaultdpiBar_Scroll); + // // MainForm // this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F); this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; this.ClientSize = new System.Drawing.Size(905, 334); this.ContextMenuStrip = this.contextMenu; + this.Controls.Add(this.defaultdpiLabel); + this.Controls.Add(this.label2); + this.Controls.Add(this.defaultdpiBar); this.Controls.Add(this.voiceCombo); this.Controls.Add(this.voiceComboLabel); this.Controls.Add(this.settingsLabel); @@ -387,6 +423,7 @@ private void InitializeComponent() ((System.ComponentModel.ISupportInitialize)(this.processedImage)).EndInit(); ((System.ComponentModel.ISupportInitialize)(this.distanceBar)).EndInit(); this.contextMenu.ResumeLayout(false); + ((System.ComponentModel.ISupportInitialize)(this.defaultdpiBar)).EndInit(); this.ResumeLayout(false); this.PerformLayout(); @@ -427,6 +464,9 @@ private void InitializeComponent() private System.Windows.Forms.Label settingsLabel; private System.Windows.Forms.Label voiceComboLabel; private System.Windows.Forms.ComboBox voiceCombo; + private System.Windows.Forms.Label defaultdpiLabel; + private System.Windows.Forms.Label label2; + private System.Windows.Forms.TrackBar defaultdpiBar; } } diff --git a/MainForm.cs b/MainForm.cs index 51575a2..bb98518 100644 --- a/MainForm.cs +++ b/MainForm.cs @@ -8,11 +8,10 @@ namespace GameOCRTTS { public partial class MainForm : Form { - private KeyboardHook _Hook = new KeyboardHook(); - private Color _Brightest = Color.White; - private int _FadeDistance = 15; + private KeyboardHook _Hook = new KeyboardHook(); private static readonly string GithubUsername = "MrFlapstaart"; private LiveUpdate _LiveUpdater = new LiveUpdate(GithubUsername); + private OCR _OCR = new OCR(); public MainForm() { @@ -25,9 +24,12 @@ public MainForm() InitializeComponent(); - colorPanel.BackColor = _Brightest; - distanceBar.Value = _FadeDistance; - distanceLabel.Text = _FadeDistance.ToString(); + colorPanel.BackColor = _OCR.Brightest; + distanceBar.Value = _OCR.FadeDistance; + distanceLabel.Text = _OCR.FadeDistance.ToString(); + + defaultdpiBar.Value = _OCR.DefaultScaleDPI; + defaultdpiLabel.Text = _OCR.DefaultScaleDPI.ToString(); voiceCombo.Items.AddRange(TTS.GetVoices().ToArray()); voiceCombo.SelectedIndex = 0; @@ -41,7 +43,7 @@ private void Hook_KeyPressed(object sender, KeyPressedEventArgs e) if (e.Modifier == SpecialKeys.Control) { Color color = ImageProc.GetColorFromCurrentPixel(); - _Brightest = color; + _OCR.Brightest = color; colorPanel.BackColor = color; SFXPlayer.PlayOK(); return; @@ -81,9 +83,10 @@ private void ocrButton_Click(object sender, EventArgs e) private void ProcessImage(Bitmap bitmap, bool forcefullscale) { - OCRResult result = OCR.HandleOCR(bitmap, _Brightest, _FadeDistance, forcefullscale); - Image resultimage = result.ProcessedImage; + OCRResult result = _OCR.HandleOCR(bitmap, forcefullscale); + Image resultimage = result.ProcessedImage; ocrBox.Text = result.ResultText; + Logger.AddLog("Original text: " + result.OriginalText); if (processedImage.Image != null) processedImage.Image.Dispose(); @@ -132,19 +135,20 @@ private void garbageButton_Click(object sender, EventArgs e) private void selectColorButton_Click(object sender, EventArgs e) { - colorSelect.Color = _Brightest; + colorSelect.Color = _OCR.Brightest; if (colorSelect.ShowDialog() != DialogResult.OK) return; - _Brightest = colorSelect.Color; - colorPanel.BackColor = _Brightest; + _OCR.Brightest = colorSelect.Color; + colorPanel.BackColor = _OCR.Brightest; } private void distanceBar_Scroll(object sender, EventArgs e) { - _FadeDistance = distanceBar.Value; - distanceLabel.Text = _FadeDistance.ToString(); + _OCR.FadeDistance = distanceBar.Value; + distanceLabel.Text = _OCR.FadeDistance.ToString(); } + // Context menu links. private void contextMenuHelp_Click(object sender, EventArgs e) { @@ -207,6 +211,12 @@ private void contextMenuAbout_Click(object sender, EventArgs e) { MessageBox.Show($"{_LiveUpdater.Product} version {_LiveUpdater.CurrentVersion} by @MrFlapstaart and @wrt54g", "About", MessageBoxButtons.OK, MessageBoxIcon.Information); } + + private void defaultdpiBar_Scroll(object sender, EventArgs e) + { + _OCR.DefaultScaleDPI = defaultdpiBar.Value; + defaultdpiLabel.Text = defaultdpiBar.Value.ToString(); + } // End of context menu links. } } \ No newline at end of file diff --git a/OCR.cs b/OCR.cs index ab1ab85..38b7de1 100644 --- a/OCR.cs +++ b/OCR.cs @@ -8,25 +8,31 @@ namespace GameOCRTTS { - internal static class OCR - { - private static TesseractEngine _Engine = new TesseractEngine(@".\tessdata", "eng", EngineMode.Default); - - internal static OCRResult HandleOCR(Bitmap bitmap, Color brightest, int fadedistance, bool forcefullscale) + public class OCR + { + private TesseractEngine _Engine = new TesseractEngine(@".\tessdata", "eng", EngineMode.Default); + + public Color Brightest { get; set; } = Color.White; + public int FadeDistance { get; set; } = 15; + public int DefaultScaleDPI { get; set; } = 125; + public int UpscaledDPI { get; set; } = 300; + public int UpscaleWidth { get; set; } = 1024; + + public OCRResult HandleOCR(Bitmap bitmap, bool forcefullscale) { Logger.AddLog("Rescaling image."); Image resultimage; bool fullscale = false; - if (bitmap.Width <= 1024 || forcefullscale) + if (bitmap.Width <= UpscaleWidth || forcefullscale) { - resultimage = ImageProc.Rescale(bitmap, 300, 300); + resultimage = ImageProc.Rescale(bitmap, UpscaledDPI, UpscaledDPI); fullscale = true; } else - resultimage = ImageProc.Rescale(bitmap, 125, 125); + resultimage = ImageProc.Rescale(bitmap, DefaultScaleDPI, DefaultScaleDPI); Logger.AddLog("Stripping colors from image."); - resultimage = ImageProc.StripColorsFromImage(resultimage, brightest, fadedistance); + resultimage = ImageProc.StripColorsFromImage(resultimage, Brightest, FadeDistance); Logger.AddLog("Handle OCR."); TextBlock block = GetTextFromImage(resultimage); @@ -36,7 +42,7 @@ internal static OCRResult HandleOCR(Bitmap bitmap, Color brightest, int fadedist { Logger.AddLog("No result, retrying on full scale."); resultimage = ImageProc.Rescale(bitmap, 300, 300); - resultimage = ImageProc.StripColorsFromImage(resultimage, brightest, fadedistance); + resultimage = ImageProc.StripColorsFromImage(resultimage, Brightest, FadeDistance); block = GetTextFromImage(resultimage); resulttext = GetProcessedTextFromBlock(block); } @@ -53,34 +59,87 @@ internal static OCRResult HandleOCR(Bitmap bitmap, Color brightest, int fadedist OCRResult result = new OCRResult(); result.Block = block; result.ResultText = resulttext; + result.OriginalText = result.OriginalText; result.ProcessedImage = resultimage; return result; } - internal static string GetProcessedTextFromBlock(TextBlock block) + private static string GetProcessedTextFromBlock(TextBlock block) { string stripped = TextHelper.StripSpecialCharacters(block.Text); string result = TextHelper.RemoveGarbageText(stripped); return result; } - - internal static TextBlock GetTextFromImage(Image image) + private TextBlock GetTextFromImage(Image image) { MemoryStream byteStream = new MemoryStream(); - image.Save(byteStream, System.Drawing.Imaging.ImageFormat.Tiff); + //image.Save(byteStream, System.Drawing.Imaging.ImageFormat.Tiff); + //byteStream.Position = 0; + Rect region = new Rect(0, 0, image.Width, image.Height); + //TesseractResult tessresult = DoTesseractByTiffStream(byteStream.ToArray(), region); + TesseractResult tessresult = DoTesseractByImage(image, region); + TextBlock result = CleanupTextBlocks(tessresult); byteStream.Position = 0; - TextBlock result = GetTextFromTiffStream(byteStream.ToArray()); + if (result.HPos > 50 && result.Height > 1) + { + //image = ImageProc.CropImage(image, new Rectangle(result.HPos, result.VPos, result.Width, result.Height)); + //image = ImageProc.Rescale(image, 300, 300); + //image.Save(@"c:\temp\parttest.png"); + region = new Rect(result.HPos - 50, result.VPos - 50, result.Width + 50, result.Height + 50); + tessresult = DoTesseractByTiffStream(byteStream.ToArray(), region); + tessresult = DoTesseractByImage(image, new Rect(0, 0, image.Width, image.Height)); + result = CleanupTextBlocks(tessresult); + } + result.OriginalText = tessresult.OriginalText; return result; } - internal static TextBlock GetTextFromTiffStream(byte[] image) + private TextBlock CleanupTextBlocks(TesseractResult result) + { + if ((result?.PrintSpace?.ComposedBlock?.Count ?? 0) == 0) + throw new Exception(""); + + ComposedBlock cblock = result.PrintSpace.ComposedBlock.OrderByDescending(x => x.WordsInComposedBlock).FirstOrDefault() ?? new ComposedBlock(); + TextBlock block = cblock.Blocks.OrderByDescending(x => x.WordsInBlock).FirstOrDefault() ?? new TextBlock(); + TextBlock orgblock = new TextBlock(); + orgblock.Lines.AddRange(block.Lines); + + int linenum = 1; + foreach (var proccblock in result.PrintSpace.ComposedBlock.OrderBy(x => x.Id)) + { + foreach (var procblock in proccblock.Blocks.OrderBy(x => x.Id)) + { + TextHelper.ProcessTextBlock(procblock); + foreach (var procline in procblock.Lines.OrderBy(x => x.Id)) + { + if (procline.Id.Contains("_")) + procline.Id = $"line{linenum++:0000}"; + if (procblock.Id != block.Id) + { + int eol = procline.HPos + procline.Width; + if ((eol >= block.HPos && eol <= block.HPos + block.Width + 50) || + (procblock.VPos >= block.VPos - 10 && procblock.VPos <= block.VPos + 10)) + block.Lines.Add(procline); + } + } + } + } + + TextHelper.ProcessTextBlock(block); + if (block.Lines.Count > 0) + ResetBlockBoundaries(block); + + return block; + } + + private TesseractResult DoTesseractByTiffStream(byte[] image, Rect region) { try { using (var img = Pix.LoadFromMemory(image)) - { - using (var page = _Engine.Process(img, PageSegMode.Auto)) + { + using (var page = _Engine.Process(img, region, PageSegMode.Auto)) { string text = page.GetText()?.Replace("\n", " "); @@ -92,47 +151,49 @@ internal static TextBlock GetTextFromTiffStream(byte[] image) result = (TesseractResult)serializer.Deserialize(reader); } - ComposedBlock cblock = result.PrintSpace.ComposedBlock.OrderByDescending(x => x.WordsInComposedBlock).FirstOrDefault() ?? new ComposedBlock(); - TextBlock block = cblock.Blocks.OrderByDescending(x => x.WordsInBlock).FirstOrDefault() ?? new TextBlock(); - TextBlock orgblock = new TextBlock(); - orgblock.Lines.AddRange(block.Lines); - - int linenum = 1; - foreach (var proccblock in result.PrintSpace.ComposedBlock.OrderBy(x => x.Id)) - { - foreach (var procblock in proccblock.Blocks.OrderBy(x => x.Id)) - { - TextHelper.ProcessTextBlock(procblock); - foreach (var procline in procblock.Lines.OrderBy(x => x.Id)) - { - if (procline.Id.Contains("_")) - procline.Id = $"line{linenum++:0000}"; - if (procblock.Id != block.Id) - { - int eol = procline.HPos + procline.Width; - if ((eol >= block.HPos && eol <= block.HPos + block.Width + 50) || - (procblock.VPos >= block.VPos - 10 && procblock.VPos <= block.VPos + 10)) - block.Lines.Add(procline); - } - } - } - } - - TextHelper.ProcessTextBlock(block); - if (block.Lines.Count > 0) - ResetBlockBoundaries(block); - - return block; + result.OriginalText = text; + return result; } } } catch (Exception ex) { Trace.TraceError($"Error: {ex.Message}\nStack: {ex.StackTrace}"); - return new TextBlock() { Text = $"Error: {ex.Message}" }; + return new TesseractResult() { Result = $"Error: {ex.Message}" }; } } + private TesseractResult DoTesseractByImage(Image image, Rect region) + { + try + { + Bitmap bitmap = new Bitmap(image); + + using (var page = _Engine.Process(bitmap, region, PageSegMode.Auto)) + { + string text = page.GetText()?.Replace("\n", " "); + + string xml = page.GetAltoText(1); + var serializer = new XmlSerializer(typeof(TesseractResult)); + TesseractResult result = new TesseractResult(); + using (TextReader reader = new StringReader(xml)) + { + result = (TesseractResult)serializer.Deserialize(reader); + } + + result.OriginalText = text; + return result; + } + + } + catch (Exception ex) + { + Trace.TraceError($"Error: {ex.Message}\nStack: {ex.StackTrace}"); + return new TesseractResult() { Result = $"Error: {ex.Message}" }; + } + } + + private static void ResetBlockBoundaries(TextBlock block) { foreach (var line in block.Lines) diff --git a/OCRResult.cs b/OCRResult.cs index e27732b..bce1f46 100644 --- a/OCRResult.cs +++ b/OCRResult.cs @@ -94,6 +94,9 @@ public int WordsInBlock set { } } + [XmlIgnore] + public string OriginalText { get; set; } + private string _Text; [XmlIgnore] public string Text @@ -105,7 +108,7 @@ public string Text } set { _Text = value; } } - + public override string ToString() { return $"{HPos},{VPos},{Width},{Height} : {Text}"; @@ -164,12 +167,16 @@ public class TesseractResult [XmlIgnore] public string Result { get; set; } + + [XmlIgnore] + public string OriginalText { get; set; } } public class OCRResult { public TextBlock Block { get; set; } public Image ProcessedImage { get; set; } + public string OriginalText { get; set; } public string ResultText { get; set; } } } diff --git a/Properties/AssemblyInfo.cs b/Properties/AssemblyInfo.cs index e0bd3e1..f6042c3 100644 --- a/Properties/AssemblyInfo.cs +++ b/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.4.0.0")] -[assembly: AssemblyFileVersion("1.4.0.0")] +[assembly: AssemblyVersion("1.5.0.0")] +[assembly: AssemblyFileVersion("1.5.0.0")]