Skip to content

edsnlp.pipelines.core.endlines.endlines

EndLines

Bases: GenericMatcher

spaCy Pipeline to detect whether a newline character should be considered a space (ie introduced by the PDF).

The pipeline will add the extension end_line to spans and tokens. The end_line attribute is a boolean or None, set to True if the pipeline predicts that the new line is an end line character. Otherwise, it is set to False if the new line is classified as a space. If no classification has been done over that token, it will remain None.

PARAMETER DESCRIPTION
nlp

spaCy nlp pipeline to use for matching.

TYPE: Language

end_lines_model : Optional[Union[str, EndLinesModel]], by default None path to trained model. If None, it will use a default model

Source code in edsnlp/pipelines/core/endlines/endlines.py 
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
class EndLines(GenericMatcher):
    """
    spaCy Pipeline to detect whether a newline character should
    be considered a space (ie introduced by the PDF).

    The pipeline will add the extension `end_line` to spans
    and tokens. The `end_line` attribute is a boolean or `None`,
    set to `True` if the pipeline predicts that the new line
    is an end line character. Otherwise, it is  set to `False`
    if the new line is classified as a space. If no classification
    has been done over that token, it will remain `None`.

    Parameters
    ----------
    nlp : Language
        spaCy nlp pipeline to use for matching.

    end_lines_model : Optional[Union[str, EndLinesModel]], by default None
        path to trained model. If None, it will use a default model
    """

    def __init__(
        self,
        nlp: Language,
        end_lines_model: Optional[Union[str, EndLinesModel]],
        **kwargs,
    ):

        super().__init__(
            nlp,
            terms=None,
            attr="TEXT",
            regex=dict(
                new_line=r"\n+",
            ),
            ignore_excluded=False,
            **kwargs,
        )

        if not Token.has_extension("end_line"):
            Token.set_extension("end_line", default=None)

        if not Span.has_extension("end_line"):
            Span.set_extension("end_line", default=None)

        self._read_model(end_lines_model)

    def _read_model(self, end_lines_model: Optional[Union[str, EndLinesModel]]):
        """
        Parameters
        ----------
        end_lines_model : Optional[Union[str, EndLinesModel]]

        Raises
        ------
        TypeError
        """
        if end_lines_model is None:
            path = build_path(__file__, "base_model.pkl")

            with open(path, "rb") as inp:
                self.model = pickle.load(inp)
        elif type(end_lines_model) == str:
            with open(end_lines_model, "rb") as inp:
                self.model = pickle.load(inp)
        elif type(end_lines_model) == EndLinesModel:
            self.model = end_lines_model
        else:
            raise TypeError(
                "type(`end_lines_model`) should be one of {None, str, EndLinesModel}"
            )

    @staticmethod
    def _spacy_compute_a3a4(token: Token) -> str:
        """Function to compute A3 and A4

        Parameters
        ----------
        token : Token

        Returns
        -------
        str
        """

        if token.is_upper:
            return "UPPER"

        elif token.shape_.startswith("Xx"):
            return "S_UPPER"

        elif token.shape_.startswith("x"):
            return "LOWER"

        elif (token.is_digit) & (
            (token.doc[max(token.i - 1, 0)].is_punct)
            | (token.doc[min(token.i + 1, len(token.doc) - 1)].is_punct)
        ):
            return "ENUMERATION"

        elif token.is_digit:
            return "DIGIT"

        elif (token.is_punct) & (token.text in [".", ";", "..", "..."]):
            return "STRONG_PUNCT"

        elif (token.is_punct) & (token.text not in [".", ";", "..", "..."]):
            return "SOFT_PUNCT"

        else:
            return "OTHER"

    @staticmethod
    def _compute_length(doc: Doc, start: int, end: int) -> int:
        """Compute length without spaces

        Parameters
        ----------
        doc : Doc
        start : int
        end : int

        Returns
        -------
        int
        """
        length = 0
        for t in doc[start:end]:
            length += len(t.text)

        return length

    def _get_df(self, doc: Doc, new_lines: List[Span]) -> pd.DataFrame:
        """Get a pandas DataFrame to call the classifier

        Parameters
        ----------
        doc : Doc
        new_lines : List[Span]

        Returns
        -------
        pd.DataFrame
        """

        data = []
        for i, span in enumerate(new_lines):
            start = span.start
            end = span.end

            max_index = len(doc) - 1
            a1_token = doc[max(start - 1, 0)]
            a2_token = doc[min(start + 1, max_index)]
            a1 = a1_token.orth
            a2 = a2_token.orth
            a3 = self._spacy_compute_a3a4(a1_token)
            a4 = self._spacy_compute_a3a4(a2_token)
            blank_line = "\n\n" in span.text

            if i > 0:
                start_previous = new_lines[i - 1].start + 1
            else:
                start_previous = 0

            length = self._compute_length(
                doc, start=start_previous, end=start
            )  # It's ok cause i count the total length from the previous up to this one

            data_dict = dict(
                span_start=start,
                span_end=end,
                A1=a1,
                A2=a2,
                A3=a3,
                A4=a4,
                BLANK_LINE=blank_line,
                length=length,
            )
            data.append(data_dict)

        df = pd.DataFrame(data)

        mu = df["length"].mean()
        sigma = df["length"].std()
        if np.isnan(sigma):
            sigma = 1

        cv = sigma / mu
        df["B1"] = (df["length"] - mu) / sigma
        df["B2"] = cv

        return df

    def __call__(self, doc: Doc) -> Doc:
        """
        Predict for each new line if it's an end of line or a space.

        Parameters
        ----------
        doc: spaCy Doc object

        Returns
        -------
        doc: spaCy Doc object, with each new line annotated
        """

        matches = self.process(doc)
        new_lines = get_spans(matches, "new_line")

        if len(new_lines) > 0:
            df = self._get_df(doc=doc, new_lines=new_lines)
            df = self.model.predict(df)

            spans = []
            for span, prediction in zip(new_lines, df.PREDICTED_END_LINE):

                span.label_ = _get_label(prediction)
                span._.end_line = prediction

                spans.append(span)
                for t in span:
                    t._.end_line = prediction
                    if not prediction:
                        t._.excluded = True

            doc.spans["new_lines"] = spans
        return doc

_read_model(end_lines_model)

PARAMETER DESCRIPTION
end_lines_model

TYPE: Optional[Union[str, EndLinesModel]]

RAISES DESCRIPTION
TypeError
Source code in edsnlp/pipelines/core/endlines/endlines.py 
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
def _read_model(self, end_lines_model: Optional[Union[str, EndLinesModel]]):
    """
    Parameters
    ----------
    end_lines_model : Optional[Union[str, EndLinesModel]]

    Raises
    ------
    TypeError
    """
    if end_lines_model is None:
        path = build_path(__file__, "base_model.pkl")

        with open(path, "rb") as inp:
            self.model = pickle.load(inp)
    elif type(end_lines_model) == str:
        with open(end_lines_model, "rb") as inp:
            self.model = pickle.load(inp)
    elif type(end_lines_model) == EndLinesModel:
        self.model = end_lines_model
    else:
        raise TypeError(
            "type(`end_lines_model`) should be one of {None, str, EndLinesModel}"
        )

_spacy_compute_a3a4(token)

Function to compute A3 and A4

PARAMETER DESCRIPTION
token

TYPE: Token

RETURNS DESCRIPTION
str
Source code in edsnlp/pipelines/core/endlines/endlines.py 
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
@staticmethod
def _spacy_compute_a3a4(token: Token) -> str:
    """Function to compute A3 and A4

    Parameters
    ----------
    token : Token

    Returns
    -------
    str
    """

    if token.is_upper:
        return "UPPER"

    elif token.shape_.startswith("Xx"):
        return "S_UPPER"

    elif token.shape_.startswith("x"):
        return "LOWER"

    elif (token.is_digit) & (
        (token.doc[max(token.i - 1, 0)].is_punct)
        | (token.doc[min(token.i + 1, len(token.doc) - 1)].is_punct)
    ):
        return "ENUMERATION"

    elif token.is_digit:
        return "DIGIT"

    elif (token.is_punct) & (token.text in [".", ";", "..", "..."]):
        return "STRONG_PUNCT"

    elif (token.is_punct) & (token.text not in [".", ";", "..", "..."]):
        return "SOFT_PUNCT"

    else:
        return "OTHER"

_compute_length(doc, start, end)

Compute length without spaces

PARAMETER DESCRIPTION
doc

TYPE: Doc

start

TYPE: int

end

TYPE: int

RETURNS DESCRIPTION
int
Source code in edsnlp/pipelines/core/endlines/endlines.py 
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
@staticmethod
def _compute_length(doc: Doc, start: int, end: int) -> int:
    """Compute length without spaces

    Parameters
    ----------
    doc : Doc
    start : int
    end : int

    Returns
    -------
    int
    """
    length = 0
    for t in doc[start:end]:
        length += len(t.text)

    return length

_get_df(doc, new_lines)

Get a pandas DataFrame to call the classifier

PARAMETER DESCRIPTION
doc

TYPE: Doc

new_lines

TYPE: List[Span]

RETURNS DESCRIPTION
pd.DataFrame
Source code in edsnlp/pipelines/core/endlines/endlines.py 
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
def _get_df(self, doc: Doc, new_lines: List[Span]) -> pd.DataFrame:
    """Get a pandas DataFrame to call the classifier

    Parameters
    ----------
    doc : Doc
    new_lines : List[Span]

    Returns
    -------
    pd.DataFrame
    """

    data = []
    for i, span in enumerate(new_lines):
        start = span.start
        end = span.end

        max_index = len(doc) - 1
        a1_token = doc[max(start - 1, 0)]
        a2_token = doc[min(start + 1, max_index)]
        a1 = a1_token.orth
        a2 = a2_token.orth
        a3 = self._spacy_compute_a3a4(a1_token)
        a4 = self._spacy_compute_a3a4(a2_token)
        blank_line = "\n\n" in span.text

        if i > 0:
            start_previous = new_lines[i - 1].start + 1
        else:
            start_previous = 0

        length = self._compute_length(
            doc, start=start_previous, end=start
        )  # It's ok cause i count the total length from the previous up to this one

        data_dict = dict(
            span_start=start,
            span_end=end,
            A1=a1,
            A2=a2,
            A3=a3,
            A4=a4,
            BLANK_LINE=blank_line,
            length=length,
        )
        data.append(data_dict)

    df = pd.DataFrame(data)

    mu = df["length"].mean()
    sigma = df["length"].std()
    if np.isnan(sigma):
        sigma = 1

    cv = sigma / mu
    df["B1"] = (df["length"] - mu) / sigma
    df["B2"] = cv

    return df

__call__(doc)

Predict for each new line if it's an end of line or a space.

PARAMETER DESCRIPTION
doc

TYPE: Doc

RETURNS DESCRIPTION
doc
Source code in edsnlp/pipelines/core/endlines/endlines.py 
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
def __call__(self, doc: Doc) -> Doc:
    """
    Predict for each new line if it's an end of line or a space.

    Parameters
    ----------
    doc: spaCy Doc object

    Returns
    -------
    doc: spaCy Doc object, with each new line annotated
    """

    matches = self.process(doc)
    new_lines = get_spans(matches, "new_line")

    if len(new_lines) > 0:
        df = self._get_df(doc=doc, new_lines=new_lines)
        df = self.model.predict(df)

        spans = []
        for span, prediction in zip(new_lines, df.PREDICTED_END_LINE):

            span.label_ = _get_label(prediction)
            span._.end_line = prediction

            spans.append(span)
            for t in span:
                t._.end_line = prediction
                if not prediction:
                    t._.excluded = True

        doc.spans["new_lines"] = spans
    return doc
Back to top