edsnlp.matchers.regex

RegexMatcher

Bases: object

Simple RegExp matcher.

PARAMETER	DESCRIPTION
alignment_mode	How spans should be aligned with tokens. Possible values are strict (character indices must be aligned with token boundaries), "contract" (span of all tokens completely within the character span), "expand" (span of all tokens at least partially covered by the character span). Defaults to expand. TYPE: str
attr	Default attribute to match on, by default "TEXT". Can be overiden in the add method. TYPE: str
flags	Additional flags provided to the re module. Can be overiden in the add method. TYPE: Union[re.RegexFlag, int]
ignore_excluded	Whether to skip exclusions TYPE: bool
span_from_group	If set to False, will create spans basede on the regex's full match. If set to True, will use the first matching capturing group as a span (and fall back to using the full match if no capturing group is matching) TYPE: bool

Source code in edsnlp/matchers/regex.py

class RegexMatcher(object):
    """
    Simple RegExp matcher.

    Parameters
    ----------
    alignment_mode : str
        How spans should be aligned with tokens.
        Possible values are `strict` (character indices must be aligned
        with token boundaries), "contract" (span of all tokens completely
        within the character span), "expand" (span of all tokens at least
        partially covered by the character span).
        Defaults to `expand`.
    attr : str
        Default attribute to match on, by default "TEXT".
        Can be overiden in the `add` method.
    flags : Union[re.RegexFlag, int]
        Additional flags provided to the `re` module.
        Can be overiden in the `add` method.
    ignore_excluded : bool
        Whether to skip exclusions
    span_from_group : bool
        If set to `False`, will create spans basede on the regex's full match.
        If set to `True`, will use the first matching capturing group as a span
        (and fall back to using the full match if no capturing group is matching)
    """

    def __init__(
        self,
        alignment_mode: str = "expand",
        attr: str = "TEXT",
        ignore_excluded: bool = False,
        flags: Union[re.RegexFlag, int] = 0,  # No additional flags
        span_from_group: bool = False,
    ):
        self.alignment_mode = alignment_mode
        self.regex = []

        self.default_attr = attr

        self.flags = flags
        self.span_from_group = span_from_group

        self.ignore_excluded = ignore_excluded

        self.set_extensions()

    @staticmethod
    def set_extensions():
        if not Span.has_extension("normalized_variant"):
            Span.set_extension("normalized_variant", getter=get_normalized_variant)

    def build_patterns(self, regex: Patterns):
        """
        Build patterns and adds them for matching.
        Helper function for pipelines using this matcher.

        Parameters
        ----------
        regex : Patterns
            Dictionary of label/terms, or label/dictionary of terms/attribute.
        """
        if not regex:
            regex = dict()

        for key, patterns in regex.items():
            if isinstance(patterns, dict):
                attr = patterns.get("attr")
                alignment_mode = patterns.get("alignment_mode")
                flags = patterns.get("flags")
                patterns = patterns.get("regex")
            else:
                attr = None
                alignment_mode = None
                flags = None

            if isinstance(patterns, str):
                patterns = [patterns]

            self.add(
                key=key,
                patterns=patterns,
                attr=attr,
                alignment_mode=alignment_mode,
                flags=flags,
            )

    def add(
        self,
        key: str,
        patterns: List[str],
        attr: Optional[str] = None,
        ignore_excluded: Optional[bool] = None,
        alignment_mode: Optional[str] = None,
        flags: Optional[re.RegexFlag] = None,
    ):
        """
        Add a pattern to the registry.

        Parameters
        ----------
        key : str
            Key of the new/updated pattern.
        patterns : List[str]
            List of patterns to add.
        attr : str, optional
            Attribute to use for matching.
            By default uses the `default_attr` attribute
        ignore_excluded : bool, optional
            Whether to skip excluded tokens during matching.
        alignment_mode : str, optional
            Overwrite alignment mode.
        """

        if attr is None:
            attr = self.default_attr

        if ignore_excluded is None:
            ignore_excluded = self.ignore_excluded

        if alignment_mode is None:
            alignment_mode = self.alignment_mode

        if flags is None:
            flags = self.flags

        patterns = [compile_regex(pattern, flags) for pattern in patterns]

        self.regex.append((key, patterns, attr, ignore_excluded, alignment_mode))

    def remove(
        self,
        key: str,
    ):
        """
        Remove a pattern for the registry.

        Parameters
        ----------
        key : str
            key of the pattern to remove.

        Raises
        ------
        ValueError
            If the key is not present in the registered patterns.
        """
        n = len(self.regex)
        self.regex = [(k, p, a, i, am) for k, p, a, i, am in self.regex if k != key]
        if len(self.regex) == n:
            raise ValueError(f"`{key}` is not referenced in the matcher")

    def __len__(self):
        return len(set([regex[0] for regex in self.regex]))

    def match(
        self,
        doclike: Union[Doc, Span],
    ) -> Tuple[Span, re.Match]:
        """
        Iterates on the matches.

        Parameters
        ----------
        doclike:
            spaCy Doc or Span object to match on.

        Yields
        -------
        span:
            A match.
        """

        for key, patterns, attr, ignore_excluded, alignment_mode in self.regex:
            text = get_text(doclike, attr, ignore_excluded)

            for pattern in patterns:
                for match in pattern.finditer(text):
                    logger.trace(f"Matched a regex from {key}: {repr(match.group())}")

                    start_char, end_char = span_from_match(
                        match=match,
                        span_from_group=self.span_from_group,
                    )

                    span = create_span(
                        doclike=doclike,
                        start_char=start_char,
                        end_char=end_char,
                        key=key,
                        attr=attr,
                        alignment_mode=alignment_mode,
                        ignore_excluded=ignore_excluded,
                    )

                    if span is None:
                        continue

                    yield span, match

    def __call__(
        self,
        doclike: Union[Doc, Span],
        as_spans=False,
        return_groupdict=False,
    ) -> Union[Span, Tuple[Span, Dict[str, Any]]]:
        """
        Performs matching. Yields matches.

        Parameters
        ----------
        doclike:
            spaCy Doc or Span object.
        as_spans:
            Returns matches as spans.

        Yields
        ------
        span:
            A match.
        groupdict:
            Additional information coming from the named patterns
            in the regular expression.
        """
        for span, match in self.match(doclike):
            if not as_spans:
                offset = doclike[0].i
                span = (span.label, span.start - offset, span.end - offset)
            if return_groupdict:
                yield span, match.groupdict()
            else:
                yield span

alignment_mode = alignment_mode instance-attribute

regex = [] instance-attribute

default_attr = attr instance-attribute

flags = flags instance-attribute

span_from_group = span_from_group instance-attribute

ignore_excluded = ignore_excluded instance-attribute

__init__(alignment_mode='expand', attr='TEXT', ignore_excluded=False, flags=0, span_from_group=False)

Source code in edsnlp/matchers/regex.py

def __init__(
    self,
    alignment_mode: str = "expand",
    attr: str = "TEXT",
    ignore_excluded: bool = False,
    flags: Union[re.RegexFlag, int] = 0,  # No additional flags
    span_from_group: bool = False,
):
    self.alignment_mode = alignment_mode
    self.regex = []

    self.default_attr = attr

    self.flags = flags
    self.span_from_group = span_from_group

    self.ignore_excluded = ignore_excluded

    self.set_extensions()

set_extensions()

Source code in edsnlp/matchers/regex.py

@staticmethod
def set_extensions():
    if not Span.has_extension("normalized_variant"):
        Span.set_extension("normalized_variant", getter=get_normalized_variant)

build_patterns(regex)

Build patterns and adds them for matching. Helper function for pipelines using this matcher.

PARAMETER	DESCRIPTION
regex	Dictionary of label/terms, or label/dictionary of terms/attribute. TYPE: Patterns

Source code in edsnlp/matchers/regex.py

def build_patterns(self, regex: Patterns):
    """
    Build patterns and adds them for matching.
    Helper function for pipelines using this matcher.

    Parameters
    ----------
    regex : Patterns
        Dictionary of label/terms, or label/dictionary of terms/attribute.
    """
    if not regex:
        regex = dict()

    for key, patterns in regex.items():
        if isinstance(patterns, dict):
            attr = patterns.get("attr")
            alignment_mode = patterns.get("alignment_mode")
            flags = patterns.get("flags")
            patterns = patterns.get("regex")
        else:
            attr = None
            alignment_mode = None
            flags = None

        if isinstance(patterns, str):
            patterns = [patterns]

        self.add(
            key=key,
            patterns=patterns,
            attr=attr,
            alignment_mode=alignment_mode,
            flags=flags,
        )

add(key, patterns, attr=None, ignore_excluded=None, alignment_mode=None, flags=None)

Add a pattern to the registry.

PARAMETER	DESCRIPTION
key	Key of the new/updated pattern. TYPE: str
patterns	List of patterns to add. TYPE: List[str]
attr	Attribute to use for matching. By default uses the default_attr attribute TYPE: str, optional DEFAULT: None
ignore_excluded	Whether to skip excluded tokens during matching. TYPE: bool, optional DEFAULT: None
alignment_mode	Overwrite alignment mode. TYPE: str, optional DEFAULT: None

Source code in edsnlp/matchers/regex.py

def add(
    self,
    key: str,
    patterns: List[str],
    attr: Optional[str] = None,
    ignore_excluded: Optional[bool] = None,
    alignment_mode: Optional[str] = None,
    flags: Optional[re.RegexFlag] = None,
):
    """
    Add a pattern to the registry.

    Parameters
    ----------
    key : str
        Key of the new/updated pattern.
    patterns : List[str]
        List of patterns to add.
    attr : str, optional
        Attribute to use for matching.
        By default uses the `default_attr` attribute
    ignore_excluded : bool, optional
        Whether to skip excluded tokens during matching.
    alignment_mode : str, optional
        Overwrite alignment mode.
    """

    if attr is None:
        attr = self.default_attr

    if ignore_excluded is None:
        ignore_excluded = self.ignore_excluded

    if alignment_mode is None:
        alignment_mode = self.alignment_mode

    if flags is None:
        flags = self.flags

    patterns = [compile_regex(pattern, flags) for pattern in patterns]

    self.regex.append((key, patterns, attr, ignore_excluded, alignment_mode))

remove(key)

Remove a pattern for the registry.

PARAMETER	DESCRIPTION
key	key of the pattern to remove. TYPE: str

RAISES	DESCRIPTION
ValueError	If the key is not present in the registered patterns.

Source code in edsnlp/matchers/regex.py

def remove(
    self,
    key: str,
):
    """
    Remove a pattern for the registry.

    Parameters
    ----------
    key : str
        key of the pattern to remove.

    Raises
    ------
    ValueError
        If the key is not present in the registered patterns.
    """
    n = len(self.regex)
    self.regex = [(k, p, a, i, am) for k, p, a, i, am in self.regex if k != key]
    if len(self.regex) == n:
        raise ValueError(f"`{key}` is not referenced in the matcher")

__len__()

Source code in edsnlp/matchers/regex.py

def __len__(self):
    return len(set([regex[0] for regex in self.regex]))

match(doclike)

Iterates on the matches.

PARAMETER	DESCRIPTION
doclike	spaCy Doc or Span object to match on. TYPE: Union[Doc, Span]

YIELDS	DESCRIPTION
span	A match.

Source code in edsnlp/matchers/regex.py

def match(
    self,
    doclike: Union[Doc, Span],
) -> Tuple[Span, re.Match]:
    """
    Iterates on the matches.

    Parameters
    ----------
    doclike:
        spaCy Doc or Span object to match on.

    Yields
    -------
    span:
        A match.
    """

    for key, patterns, attr, ignore_excluded, alignment_mode in self.regex:
        text = get_text(doclike, attr, ignore_excluded)

        for pattern in patterns:
            for match in pattern.finditer(text):
                logger.trace(f"Matched a regex from {key}: {repr(match.group())}")

                start_char, end_char = span_from_match(
                    match=match,
                    span_from_group=self.span_from_group,
                )

                span = create_span(
                    doclike=doclike,
                    start_char=start_char,
                    end_char=end_char,
                    key=key,
                    attr=attr,
                    alignment_mode=alignment_mode,
                    ignore_excluded=ignore_excluded,
                )

                if span is None:
                    continue

                yield span, match

__call__(doclike, as_spans=False, return_groupdict=False)

Performs matching. Yields matches.

PARAMETER	DESCRIPTION
doclike	spaCy Doc or Span object. TYPE: Union[Doc, Span]
as_spans	Returns matches as spans. DEFAULT: False

YIELDS	DESCRIPTION
span	A match.
groupdict	Additional information coming from the named patterns in the regular expression.

Source code in edsnlp/matchers/regex.py

def __call__(
    self,
    doclike: Union[Doc, Span],
    as_spans=False,
    return_groupdict=False,
) -> Union[Span, Tuple[Span, Dict[str, Any]]]:
    """
    Performs matching. Yields matches.

    Parameters
    ----------
    doclike:
        spaCy Doc or Span object.
    as_spans:
        Returns matches as spans.

    Yields
    ------
    span:
        A match.
    groupdict:
        Additional information coming from the named patterns
        in the regular expression.
    """
    for span, match in self.match(doclike):
        if not as_spans:
            offset = doclike[0].i
            span = (span.label, span.start - offset, span.end - offset)
        if return_groupdict:
            yield span, match.groupdict()
        else:
            yield span

get_first_included(doclike)

Source code in edsnlp/matchers/regex.py

@lru_cache(32)
def get_first_included(doclike: Union[Doc, Span]) -> Token:
    for token in doclike:
        if token.tag_ != "EXCLUDED":
            return token
    raise IndexError("The provided Span does not include any token")

get_normalized_variant(doclike)

Source code in edsnlp/matchers/regex.py

def get_normalized_variant(doclike) -> str:
    tokens = [t.text + t.whitespace_ for t in doclike if not t._.excluded]
    variant = "".join(tokens)
    variant = variant.rstrip(" ")
    variant = re.sub(r"\s+", " ", variant)
    return variant

spans_generator(match)

Iterates over every group, and then yields the full match

PARAMETER	DESCRIPTION
match	A match object TYPE: re.Match

YIELDS	DESCRIPTION
Tuple[int, int]	A tuple containing the start and end of the group or match

Source code in edsnlp/matchers/regex.py

def spans_generator(match: re.Match) -> Tuple[int, int]:
    """
    Iterates over every group, and then yields the full match

    Parameters
    ----------
    match : re.Match
        A match object

    Yields
    ------
    Tuple[int, int]
        A tuple containing the start and end of the group or match
    """
    for idx in range(1, len(match.groups()) + 1):
        yield match.start(idx), match.end(idx)
    yield match.start(0), match.end(0)

span_from_match(match, span_from_group)

Return the span (as a (start, end) tuple) of the first matching group. If span_from_group=True, returns the full match instead.

PARAMETER	DESCRIPTION
match	The Match object TYPE: re.Match
span_from_group	Whether to work on groups or on the full match TYPE: bool

RETURNS	DESCRIPTION
Tuple[int, int]	A tuple containing the start and end of the group or match

Source code in edsnlp/matchers/regex.py

def span_from_match(
    match: re.Match,
    span_from_group: bool,
) -> Tuple[int, int]:
    """
    Return the span (as a (start, end) tuple) of the first matching group.
    If `span_from_group=True`, returns the full match instead.

    Parameters
    ----------
    match : re.Match
        The Match object
    span_from_group : bool
        Whether to work on groups or on the full match

    Returns
    -------
    Tuple[int, int]
        A tuple containing the start and end of the group or match
    """
    if not span_from_group:
        start_char, end_char = match.start(), match.end()
    else:
        start_char, end_char = next(filter(lambda x: x[0] >= 0, spans_generator(match)))
    return start_char, end_char

create_span(doclike, start_char, end_char, key, attr, alignment_mode, ignore_excluded)

spaCy only allows strict alignment mode for char_span on Spans. This method circumvents this.

PARAMETER	DESCRIPTION
doclike	Doc or Span. TYPE: Union[Doc, Span]
start_char	Character index within the Doc-like object. TYPE: int
end_char	Character index of the end, within the Doc-like object. TYPE: int
key	The key used to match. TYPE: str
alignment_mode	The alignment mode. TYPE: str
ignore_excluded	Whether to skip excluded tokens. TYPE: bool
Returns
span	A span matched on the Doc-like object.

Source code in edsnlp/matchers/regex.py

def create_span(
    doclike: Union[Doc, Span],
    start_char: int,
    end_char: int,
    key: str,
    attr: str,
    alignment_mode: str,
    ignore_excluded: bool,
) -> Span:
    """
    spaCy only allows strict alignment mode for char_span on Spans.
    This method circumvents this.
    Parameters
    ----------
    doclike : Union[Doc, Span]
        `Doc` or `Span`.
    start_char : int
        Character index within the Doc-like object.
    end_char : int
        Character index of the end, within the Doc-like object.
    key : str
        The key used to match.
    alignment_mode : str
        The alignment mode.
    ignore_excluded : bool
        Whether to skip excluded tokens.
    Returns
    -------
    span:
        A span matched on the Doc-like object.
    """

    doc = doclike if isinstance(doclike, Doc) else doclike.doc

    # Handle the simple case immediately
    if attr in {"TEXT", "LOWER"} and not ignore_excluded:
        off = doclike[0].idx
        return doc.char_span(
            start_char + off,
            end_char + off,
            label=key,
            alignment_mode=alignment_mode,
        )

    # If doclike is a Span, we need to get the clean
    # index of the first included token
    if ignore_excluded:
        original, clean = alignment(
            doc=doc,
            attr=attr,
            ignore_excluded=ignore_excluded,
        )

        first_included = get_first_included(doclike)
        i = bisect_left(original, first_included.idx)
        first = clean[i]

    else:
        first = doclike[0].idx

    start_char = (
        first
        + start_char
        + offset(
            doc,
            attr=attr,
            ignore_excluded=ignore_excluded,
            index=first + start_char,
        )
    )

    end_char = (
        first
        + end_char
        + offset(
            doc,
            attr=attr,
            ignore_excluded=ignore_excluded,
            index=first + end_char,
        )
    )

    span = doc.char_span(
        start_char,
        end_char,
        label=key,
        alignment_mode=alignment_mode,
    )

    return span