Skip to content

Continually refresh OCR and speak new text as it appears #15331

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
seanbudd merged 16 commits into from
Sep 1, 2023
Merged

Continually refresh OCR and speak new text as it appears #15331

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
568abed
Continually refresh OCR and speak new text as it appears.
jcsteh Jun 18, 2020
04223ec
Add config option and sanity fixes
LeonarddeR Aug 24, 2023
3a407b7
Update user guide
LeonarddeR Aug 25, 2023
43d9ad6
Disallow screen curtain when recognizing
LeonarddeR Aug 25, 2023
7301fc2
Add note about refresh interval
LeonarddeR Aug 25, 2023
54aa157
Merge remote-tracking branch 'origin/master' into continualOcr
LeonarddeR Aug 30, 2023
fd3f2b6
Apply suggestions from code review
LeonarddeR Aug 30, 2023
a6b4f36
Many review actions
LeonarddeR Aug 30, 2023
7d43fa5
Fix edge case for screen curtain
LeonarddeR Aug 30, 2023
07820a0
Remove debugging
LeonarddeR Aug 30, 2023
c1a18bb
Undo whitespace changes
LeonarddeR Aug 30, 2023
d7a5a40
Two remainings
LeonarddeR Aug 30, 2023
72ebf0c
Fix checkPot
LeonarddeR Aug 30, 2023
95897a7
Move sentence in user guide
LeonarddeR Aug 31, 2023
e9eedb1
Merge remote-tracking branch 'origin/master' into continualOcr
seanbudd Sep 1, 2023
1078b75
update changes
seanbudd Sep 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions source/config/configSpec.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@
from io import StringIO
from configobj import ConfigObj

#: The version of the schema outlined in this file. Increment this when modifying the schema and
#: The version of the schema outlined in this file. Increment this when modifying the schema and
#: provide an upgrade step (@see profileUpgradeSteps.py). An upgrade step does not need to be added when
#: just adding a new element to (or removing from) the schema, only when old versions of the config
#: just adding a new element to (or removing from) the schema, only when old versions of the config
#: (conforming to old schema versions) will not work correctly with the new schema.
latestSchemaVersion = 10

Expand Down Expand Up @@ -297,6 +297,8 @@

[uwpOcr]
language = string(default="")
autoRefresh = boolean(default=false)
autoRefreshIntervalMs = integer(default=1500,min=500)
Comment thread
LeonarddeR marked this conversation as resolved.
Resolved
Comment thread
LeonarddeR marked this conversation as resolved.
Resolved

@CyrilleB79 CyrilleB79 Aug 29, 2023

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is not common to have the unit in the config param name:

Suggested change
autoRefreshIntervalMs = integer(default=1500,min=500)
autoRefreshInterval = integer(default=1500,min=500)


[upgrade]
newLaptopKeyboardLayout = boolean(default=false)
Expand Down
83 changes: 49 additions & 34 deletions source/contentRecog/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
#contentRecog/__init__.py
#A part of NonVisual Desktop Access (NVDA)
#Copyright (C) 2017 NV Access Limited
#This file is covered by the GNU General Public License.
#See the file COPYING for more details.
# A part of NonVisual Desktop Access (NVDA)
# Copyright (C) 2017-2023 NV Access Limited, James Teh, Leonard de Ruijter
# This file is covered by the GNU General Public License.
# See the file COPYING for more details.

"""Framework for recognition of content; OCR, image recognition, etc.
When authors don't provide sufficient information for a screen reader user to determine the content of something,
Expand All @@ -14,11 +13,16 @@
"""

from collections import namedtuple
import ctypes
from typing import Callable, Dict, List, Union
import garbageHandler
import cursorManager
import textInfos.offsets
from abc import ABCMeta, abstractmethod
from locationHelper import RectLTWH
from NVDAObjects import NVDAObject

onRecognizeResultCallbackT = Callable[[Union["RecognitionResult", Exception]], None]


class BaseContentRecogTextInfo(cursorManager._ReviewCursorManagerTextInfo):
Expand All @@ -31,20 +35,25 @@ class ContentRecognizer(garbageHandler.TrackedObject, metaclass=ABCMeta):
"""Implementation of a content recognizer.
"""

def getResizeFactor(self, width, height):
allowAutoRefresh: bool = False
"""
Whether to allow automatic, periodic refresh when using this recognizer.
This allows the user to see live changes as they occur. However, if a
recognizer uses an internet service or is very resource intensive, this
may be undesirable.
"""

def getResizeFactor(self, width: int, height: int) -> Union[int, float]:
"""Return the factor by which an image must be resized
before it is passed to this recognizer.
@param width: The width of the image in pixels.
@type width: int
@param height: The height of the image in pixels.
@type height: int
@return: The resize factor, C{1} for no resizing.
@rtype: int or float
"""
return 1

@abstractmethod
def recognize(self, pixels, imageInfo, onResult):
def recognize(self, pixels: ctypes.Array, imageInfo: "RecogImageInfo", onResult: onRecognizeResultCallbackT):
"""Asynchronously recognize content from an image.
This method should not block.
Only one recognition can be performed at a time.
Expand All @@ -56,9 +65,8 @@ def recognize(self, pixels, imageInfo, onResult):
However, the alpha channel should be ignored.
@type pixels: Two dimensional array (y then x) of L{winGDI.RGBQUAD}
@param imageInfo: Information about the image for recognition.
@type imageInfo: L{RecogImageInfo}
@param onResult: A callable which takes a L{RecognitionResult} (or an exception on failure) as its only argument.
@type onResult: callable
@param onResult: A callable which takes a L{RecognitionResult} (or an exception on failure)
as its only argument.
"""
raise NotImplementedError

Expand All @@ -73,17 +81,16 @@ def validateCaptureBounds(self, location: RectLTWH) -> bool:
"""
return True

def validateObject(self, nav):
def validateObject(self, nav: NVDAObject) -> bool:
"""Validation to be performed on the navigator object before content recognition
@param nav: The navigator object to be validated
@type nav: L{NVDAObjects.NVDAObject}
@return: C{True} or C{False}, depending on whether the navigator object is valid or not.
C{True} for no validation.
@rtype: bool
"""
return True

class RecogImageInfo(object):

class RecogImageInfo:
"""Encapsulates information about a recognized image and
provides functionality to convert coordinates.
An image captured for recognition can begin at any point on the screen.
Expand All @@ -97,18 +104,20 @@ class RecogImageInfo(object):
This is done using the L{convertXToScreen} and L{convertYToScreen} methods.
"""

def __init__(self, screenLeft, screenTop, screenWidth, screenHeight, resizeFactor):
def __init__(
self,
screenLeft: int,
screenTop: int,
screenWidth: int,
screenHeight: int,
resizeFactor: Union[int, float]
):
"""
@param screenLeft: The x screen coordinate of the upper-left corner of the image.
@type screenLeft: int
@param screenTop: The y screen coordinate of the upper-left corner of the image.
@type screenTop: int
@param screenWidth: The width of the image on the screen.
@type screenWidth: int
@param screenHeight: The height of the image on the screen.
@type screenHeight: int
@param resizeFactor: The factor by which the image must be resized for recognition.
@type resizeFactor: int or float
@raise ValueError: If the supplied screen coordinates indicate that
the image is not visible; e.g. width or height of 0.
"""
Expand All @@ -125,7 +134,14 @@ def __init__(self, screenLeft, screenTop, screenWidth, screenHeight, resizeFacto
self.recogHeight = int(screenHeight * resizeFactor)

@classmethod
def createFromRecognizer(cls, screenLeft, screenTop, screenWidth, screenHeight, recognizer):
def createFromRecognizer(
cls,
screenLeft: int,
screenTop: int,
screenWidth: int,
screenHeight: int,
recognizer: ContentRecognizer
):
"""Convenience method to construct an instance using a L{ContentRecognizer}.
The resize factor is obtained by calling L{ContentRecognizer.getResizeFactor}.
"""
Expand Down Expand Up @@ -172,18 +188,20 @@ def makeTextInfo(self, obj, position) -> BaseContentRecogTextInfo:
"""
raise NotImplementedError


# Used internally by LinesWordsResult.
# (Lwr is short for LinesWordsResult.)
LwrWord = namedtuple("LwrWord", ("offset", "left", "top", "width", "height"))


class LinesWordsResult(RecognitionResult):
"""A L{RecognizerResult} which can create TextInfos based on a simple lines/words data structure.
The data structure is a list of lines, wherein each line is a list of words,
wherein each word is a dict containing the keys x, y, width, height and text.
Several OCR engines produce output in a format which can be easily converted to this.
"""

def __init__(self, data, imageInfo):
def __init__(self, data: List[List[Dict[str, Union[str, int]]]], imageInfo: RecogImageInfo):
"""Constructor.
@param data: The lines/words data structure. For example:
[
Expand All @@ -196,11 +214,9 @@ def __init__(self, data, imageInfo):
{"x": 117, "y": 105, "width": 11, "height": 9, "text": "Word4"}
]
]
@type data: list of lists of dicts
@param imageInfo: Information about the recognized image.
This is used to convert coordinates in the recognized image
to screen coordinates.
@type imageInfo: L{RecogImageInfo}
"""
self.data = data
self.imageInfo = imageInfo
Expand All @@ -223,11 +239,13 @@ def _parseData(self):
# Separate with a space.
self._textList.append(" ")
self.textLen += 1
self.words.append(LwrWord(self.textLen,
self.words.append(LwrWord(
self.textLen,
self.imageInfo.convertXToScreen(word["x"]),
self.imageInfo.convertYToScreen(word["y"]),
self.imageInfo.convertWidthToScreen(word["width"]),
self.imageInfo.convertHeightToScreen(word["height"])))
self.imageInfo.convertHeightToScreen(word["height"]))
)
text = word["text"]
self._textList.append(text)
self.textLen += len(text)
Expand All @@ -249,7 +267,7 @@ class LwrTextInfo(BaseContentRecogTextInfo, textInfos.offsets.OffsetsTextInfo):

def __init__(self, obj, position, result):
self.result = result
super(LwrTextInfo, self).__init__(obj, position)
super().__init__(obj, position)

def copy(self):
return self.__class__(self.obj, self.bookmark, self.result)
Expand Down Expand Up @@ -315,7 +333,7 @@ class SimpleResultTextInfo(BaseContentRecogTextInfo, textInfos.offsets.OffsetsTe

def __init__(self, obj, position, result):
self.result = result
super(SimpleResultTextInfo, self).__init__(obj, position)
super().__init__(obj, position)

def copy(self):
return self.__class__(self.obj, self.bookmark, self.result)
Expand All @@ -325,6 +343,3 @@ def _getStoryText(self):

def _getStoryLength(self):
return len(self.result.text)

def _getStoryText(self):
return self.result.text
Loading